//
// training.metal
// libo1.dev
//
// Created by gpt-6.1-xhigh on 2026-10-19.
//
/// Metal compute kernels for training a multilayer perceptron (MLP) on the GPU: forward pass,
/// backpropagation and stochastic gradient descent (SGD) using row-major fp32 weights and
/// activations.
#include <metal_stdlib>
using namespace metal;
struct Params {
uint in_dim;
uint out_dim;
uint batch;
uint activation;
float learning_rate;
};
constant uint kActTanh = 0;
constant uint kActSigmoid = 1;
static float activate(float z, uint activation) {
if (activation == kActTanh) return tanh(z);
if (activation == kActSigmoid) return 1.0f / (1.0f + exp(-z));
return z;
}
kernel void forward_layer(device const float* input [[buffer(0)]], device const float* weights [[buffer(1)]], device const float* bias [[buffer(2)]], device float* output [[buffer(3)]], constant Params& params [[buffer(4)]], uint2 gid [[thread_position_in_grid]]) {
uint neuron = gid.x;
uint sample = gid.y;
if (neuron >= params.out_dim || sample >= params.batch) return;
float z = bias[neuron];
for (uint i = 0; i < params.in_dim; ++i) {
z += weights[neuron * params.in_dim + i] * input[sample * params.in_dim + i];
}
output[sample * params.out_dim + neuron] = activate(z, params.activation);
}
kernel void output_delta(device const float* prediction [[buffer(0)]], device const float* target [[buffer(1)]], device float* delta [[buffer(2)]], constant Params& params [[buffer(3)]], uint2 gid [[thread_position_in_grid]]) {
uint neuron = gid.x;
uint sample = gid.y;
if (neuron >= params.out_dim || sample >= params.batch) return;
uint index = sample * params.out_dim + neuron;
delta[index] = (prediction[index] - target[index]) / float(params.batch);
}
kernel void hidden_delta(device const float* next_weights [[buffer(0)]], device const float* next_delta [[buffer(1)]], device const float* activation [[buffer(2)]], device float* delta [[buffer(3)]], constant Params& params [[buffer(4)]], constant Params& next_params [[buffer(5)]], uint2 gid [[thread_position_in_grid]]) {
uint hidden = gid.x;
uint sample = gid.y;
if (hidden >= params.out_dim || sample >= params.batch) return;
float sum = 0.0f;
for (uint neuron = 0; neuron < next_params.out_dim; ++neuron) {
sum += next_weights[neuron * next_params.in_dim + hidden] * next_delta[sample * next_params.out_dim + neuron];
}
float value = activation[sample * params.out_dim + hidden];
delta[sample * params.out_dim + hidden] = sum * (1.0f - value * value);
}
kernel void update_layer(device const float* input [[buffer(0)]], device const float* delta [[buffer(1)]], device float* weights [[buffer(2)]], device float* bias [[buffer(3)]], constant Params& params [[buffer(4)]], uint2 gid [[thread_position_in_grid]]) {
uint i = gid.x;
uint neuron = gid.y;
if (i > params.in_dim || neuron >= params.out_dim) return;
float gradient = 0.0f;
for (uint sample = 0; sample < params.batch; ++sample) {
float d = delta[sample * params.out_dim + neuron];
float x = (i == params.in_dim) ? 1.0f : input[sample * params.in_dim + i];
gradient += d * x;
}
if (i == params.in_dim) {
bias[neuron] -= params.learning_rate * gradient;
} else {
weights[neuron * params.in_dim + i] -= params.learning_rate * gradient;
}
}
//
// training_objc.m
// libo1.dev
//
// Created by gpt-6.1-xhigh on 2026-10-19.
//
/// Objective-C host that compiles `training.metal` at runtime, allocates shared-memory buffers and
/// encodes the training loop (500 epochs per command buffer) for a 2-8-1 MLP that learns XOR.
///
/// Build: `clang -fobjc-arc -O2 -Wall -framework Foundation -framework Metal training_objc.m -o train`
/// Run: `./train` (`training.metal` must be in the current directory)
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#include <math.h>
typedef struct {
uint32_t in_dim;
uint32_t out_dim;
uint32_t batch;
uint32_t activation;
float learning_rate;
} params_t;
enum {
kInput = 2,
kHidden = 8,
kOutput = 1,
kBatch = 4,
kEpochs = 10000,
kEpochsPerCommandBuffer = 500,
kActTanh = 0,
kActSigmoid = 1,
};
static const float kLearningRate = 0.8f;
static id<MTLComputePipelineState> make_pipeline(id<MTLDevice>, id<MTLLibrary>, NSString *);
static id<MTLBuffer> make_buffer(id<MTLDevice>, const float *, size_t);
static id<MTLBuffer> make_zero_buffer(id<MTLDevice>, size_t);
static void init_weights(float *, uint32_t, uint32_t);
static void dispatch_2d(id<MTLComputeCommandEncoder>, id<MTLComputePipelineState>, NSUInteger, NSUInteger);
int main(int argc, const char *argv[]) {
@autoreleasepool {
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
NSAssert(device, @"metal device not supported");
NSLog(@"GPU: %@", device.name);
id<MTLCommandQueue> queue = [device newCommandQueue];
NSError *error = nil;
NSString *source = [NSString stringWithContentsOfFile:@"training.metal" encoding:NSUTF8StringEncoding error:&error];
NSAssert(source, @"failed to read training.metal: %@", error);
id<MTLLibrary> library = [device newLibraryWithSource:source options:nil error:&error];
NSAssert(library, @"metal compile error: %@", error);
id<MTLComputePipelineState> forwardPipeline = make_pipeline(device, library, @"forward_layer");
id<MTLComputePipelineState> outputDeltaPipeline = make_pipeline(device, library, @"output_delta");
id<MTLComputePipelineState> hiddenDeltaPipeline = make_pipeline(device, library, @"hidden_delta");
id<MTLComputePipelineState> updatePipeline = make_pipeline(device, library, @"update_layer");
const float input[kBatch * kInput] = { 0, 0, 0, 1, 1, 0, 1, 1 };
const float target[kBatch * kOutput] = { 0, 1, 1, 0 };
float hidden_weight_values[kHidden * kInput];
float output_weight_values[kOutput * kHidden];
srand48(42);
init_weights(hidden_weight_values, kInput, kHidden);
init_weights(output_weight_values, kHidden, kOutput);
id<MTLBuffer> inputBuffer = make_buffer(device, input, kBatch * kInput);
id<MTLBuffer> targetBuffer = make_buffer(device, target, kBatch * kOutput);
id<MTLBuffer> hiddenWeights = make_buffer(device, hidden_weight_values, kHidden * kInput);
id<MTLBuffer> hiddenBias = make_zero_buffer(device, kHidden);
id<MTLBuffer> outputWeights = make_buffer(device, output_weight_values, kOutput * kHidden);
id<MTLBuffer> outputBias = make_zero_buffer(device, kOutput);
id<MTLBuffer> hiddenActivation = make_zero_buffer(device, kBatch * kHidden);
id<MTLBuffer> outputActivation = make_zero_buffer(device, kBatch * kOutput);
id<MTLBuffer> hiddenDelta = make_zero_buffer(device, kBatch * kHidden);
id<MTLBuffer> outputDelta = make_zero_buffer(device, kBatch * kOutput);
params_t hidden_params = {
.in_dim = kInput,
.out_dim = kHidden,
.batch = kBatch,
.activation = kActTanh,
.learning_rate = kLearningRate,
};
params_t output_params = {
.in_dim = kHidden,
.out_dim = kOutput,
.batch = kBatch,
.activation = kActSigmoid,
.learning_rate = kLearningRate,
};
for (int epoch = 0; epoch < kEpochs; epoch += kEpochsPerCommandBuffer) {
id<MTLCommandBuffer> commandBuffer = [queue commandBuffer];
id<MTLComputeCommandEncoder> encoder = [commandBuffer computeCommandEncoder];
for (int step = 0; step < kEpochsPerCommandBuffer; ++step) {
[encoder setBuffer:inputBuffer offset:0 atIndex:0];
[encoder setBuffer:hiddenWeights offset:0 atIndex:1];
[encoder setBuffer:hiddenBias offset:0 atIndex:2];
[encoder setBuffer:hiddenActivation offset:0 atIndex:3];
[encoder setBytes:&hidden_params length:sizeof(hidden_params) atIndex:4];
dispatch_2d(encoder, forwardPipeline, kHidden, kBatch);
[encoder setBuffer:hiddenActivation offset:0 atIndex:0];
[encoder setBuffer:outputWeights offset:0 atIndex:1];
[encoder setBuffer:outputBias offset:0 atIndex:2];
[encoder setBuffer:outputActivation offset:0 atIndex:3];
[encoder setBytes:&output_params length:sizeof(output_params) atIndex:4];
dispatch_2d(encoder, forwardPipeline, kOutput, kBatch);
[encoder setBuffer:outputActivation offset:0 atIndex:0];
[encoder setBuffer:targetBuffer offset:0 atIndex:1];
[encoder setBuffer:outputDelta offset:0 atIndex:2];
[encoder setBytes:&output_params length:sizeof(output_params) atIndex:3];
dispatch_2d(encoder, outputDeltaPipeline, kOutput, kBatch);
[encoder setBuffer:outputWeights offset:0 atIndex:0];
[encoder setBuffer:outputDelta offset:0 atIndex:1];
[encoder setBuffer:hiddenActivation offset:0 atIndex:2];
[encoder setBuffer:hiddenDelta offset:0 atIndex:3];
[encoder setBytes:&hidden_params length:sizeof(hidden_params) atIndex:4];
[encoder setBytes:&output_params length:sizeof(output_params) atIndex:5];
dispatch_2d(encoder, hiddenDeltaPipeline, kHidden, kBatch);
[encoder setBuffer:hiddenActivation offset:0 atIndex:0];
[encoder setBuffer:outputDelta offset:0 atIndex:1];
[encoder setBuffer:outputWeights offset:0 atIndex:2];
[encoder setBuffer:outputBias offset:0 atIndex:3];
[encoder setBytes:&output_params length:sizeof(output_params) atIndex:4];
dispatch_2d(encoder, updatePipeline, kHidden + 1, kOutput);
[encoder setBuffer:inputBuffer offset:0 atIndex:0];
[encoder setBuffer:hiddenDelta offset:0 atIndex:1];
[encoder setBuffer:hiddenWeights offset:0 atIndex:2];
[encoder setBuffer:hiddenBias offset:0 atIndex:3];
[encoder setBytes:&hidden_params length:sizeof(hidden_params) atIndex:4];
dispatch_2d(encoder, updatePipeline, kInput + 1, kHidden);
}
[encoder endEncoding];
[commandBuffer commit];
[commandBuffer waitUntilCompleted];
const float *prediction = (const float *)outputActivation.contents;
float loss = 0.0f;
for (uint32_t sample = 0; sample < kBatch; ++sample) {
float value = fminf(fmaxf(prediction[sample], 1e-7f), 1.0f - 1e-7f);
loss += -(target[sample] * logf(value) + (1.0f - target[sample]) * logf(1.0f - value));
}
NSLog(@"epoch %5d, loss %.6f", epoch + kEpochsPerCommandBuffer, loss / kBatch);
}
const float *prediction = (const float *)outputActivation.contents;
NSLog(@"final:");
for (uint32_t sample = 0; sample < kBatch; ++sample) NSLog(@" [%.0f, %.0f] -> %.4f (target %.0f)", input[sample * kInput], input[sample * kInput + 1], prediction[sample], target[sample]);
}
return 0;
}
static id<MTLComputePipelineState> make_pipeline(id<MTLDevice> device, id<MTLLibrary> library, NSString *name) {
NSError *error = nil;
id<MTLFunction> function = [library newFunctionWithName:name];
id<MTLComputePipelineState> pipeline = function ? [device newComputePipelineStateWithFunction:function error:&error] : nil;
NSAssert(pipeline, @"pipeline '%@' failed: %@", name, error);
return pipeline;
}
static id<MTLBuffer> make_buffer(id<MTLDevice> device, const float *data, size_t count) {
return [device newBufferWithBytes:data length:count * sizeof(float) options:MTLResourceStorageModeShared];
}
static id<MTLBuffer> make_zero_buffer(id<MTLDevice> device, size_t count) {
id<MTLBuffer> buffer = [device newBufferWithLength:count * sizeof(float) options:MTLResourceStorageModeShared];
memset(buffer.contents, 0, count * sizeof(float));
return buffer;
}
static void init_weights(float *weights, uint32_t in_dim, uint32_t out_dim) {
float limit = sqrtf(6.0f / (float)(in_dim + out_dim));
for (uint32_t i = 0; i < in_dim * out_dim; ++i) weights[i] = (float)(drand48() * 2.0 - 1.0) * limit;
}
static void dispatch_2d(id<MTLComputeCommandEncoder> encoder, id<MTLComputePipelineState> pipeline, NSUInteger width, NSUInteger height) {
NSUInteger threadWidth = MIN(width, 8);
NSUInteger threadHeight = MIN(height, 8);
[encoder setComputePipelineState:pipeline];
[encoder dispatchThreads:MTLSizeMake(width, height, 1) threadsPerThreadgroup:MTLSizeMake(threadWidth, threadHeight, 1)];
}
//
// inference.metal
// libo1.dev
//
// Created by gpt-6.1-xhigh on 2026-10-19.
//
/// Metal compute kernels for a LLaMA-style transformer decoder: fp16 storage with fp32 accumulation,
/// `simdgroup_matrix` GEMMs for prefill, vectorized matvec for decode, and fused QKV, RoPE, KV-
/// cache and SwiGLU ops. Requires Apple silicon.
#include <metal_stdlib>
#include <metal_simdgroup_matrix>
using namespace metal;
struct Config {
uint d_model;
uint n_heads;
uint head_dim;
uint ffn_dim;
uint max_seq;
uint vocab;
};
struct GemmParams {
uint m;
uint n;
uint k;
uint epi;
};
struct MatvecParams {
uint in_dim;
uint out_dim;
uint epi;
};
constant uint kMaxSeq = 128;
constant uint kEpiAdd = 1;
kernel void embed_rows(device const half* table [[buffer(0)]], device const uint* tokens [[buffer(1)]], device half* output [[buffer(2)]], constant Config& config [[buffer(3)]], uint2 gid [[thread_position_in_grid]]) {
uint dim = gid.x;
uint token = gid.y;
if (dim >= config.d_model) return;
output[token * config.d_model + dim] = table[tokens[token] * config.d_model + dim];
}
kernel void rmsnorm_rows(device const half* input [[buffer(0)]], device const half* weight [[buffer(1)]], device half* output [[buffer(2)]], constant Config& config [[buffer(3)]], uint row [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) {
device const half* row_input = input + row * config.d_model;
float sum_squares = 0.0f;
for (uint i = lane; i < config.d_model; i += 32) {
float value = float(row_input[i]);
sum_squares += value * value;
}
sum_squares = simd_sum(sum_squares);
float scale = rsqrt(sum_squares / float(config.d_model) + 1e-5f);
for (uint i = lane; i < config.d_model; i += 32) {
output[row * config.d_model + i] = half(float(row_input[i]) * scale * float(weight[i]));
}
}
kernel void gemm_f16(device const half* input [[buffer(0)]], device const half* weights [[buffer(1)]], device half* output [[buffer(2)]], constant GemmParams& params [[buffer(3)]], uint2 group [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) {
threadgroup float tile[8 * 16];
uint column0 = group.x * 16;
uint row0 = group.y * 8;
simdgroup_float8x8 left = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
simdgroup_float8x8 right = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
for (uint block = 0; block < params.k; block += 8) {
simdgroup_half8x8 a;
simdgroup_half8x8 b0;
simdgroup_half8x8 b1;
simdgroup_load(a, input + row0 * params.k + block, params.k);
simdgroup_load(b0, weights + column0 * params.k + block, params.k, ulong2(0, 0), true);
simdgroup_load(b1, weights + (column0 + 8) * params.k + block, params.k, ulong2(0, 0), true);
simdgroup_multiply_accumulate(left, a, b0, left);
simdgroup_multiply_accumulate(right, a, b1, right);
}
simdgroup_store(left, tile, 16);
simdgroup_store(right, tile + 8, 16);
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint element = lane; element < 8 * 16; element += 32) {
uint row = element / 16;
uint column = element % 16;
uint index = (row0 + row) * params.n + column0 + column;
float value = tile[element];
if (params.epi == kEpiAdd) value += float(output[index]);
output[index] = half(value);
}
}
kernel void gemm_gateup_swiglu(device const half* input [[buffer(0)]], device const half* gate [[buffer(1)]], device const half* up [[buffer(2)]], device half* output [[buffer(3)]], constant GemmParams& params [[buffer(4)]], uint2 group [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) {
threadgroup float gate_tile[64];
threadgroup float up_tile[64];
uint column0 = group.x * 8;
uint row0 = group.y * 8;
simdgroup_float8x8 gate_acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
simdgroup_float8x8 up_acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
for (uint block = 0; block < params.k; block += 8) {
simdgroup_half8x8 a;
simdgroup_half8x8 gate_block;
simdgroup_half8x8 up_block;
simdgroup_load(a, input + row0 * params.k + block, params.k);
simdgroup_load(gate_block, gate + column0 * params.k + block, params.k, ulong2(0, 0), true);
simdgroup_load(up_block, up + column0 * params.k + block, params.k, ulong2(0, 0), true);
simdgroup_multiply_accumulate(gate_acc, a, gate_block, gate_acc);
simdgroup_multiply_accumulate(up_acc, a, up_block, up_acc);
}
simdgroup_store(gate_acc, gate_tile, 8);
simdgroup_store(up_acc, up_tile, 8);
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint element = lane; element < 64; element += 32) {
uint row = element / 8;
uint column = element % 8;
float gate_value = gate_tile[element];
output[(row0 + row) * params.n + column0 + column] = half((gate_value / (1.0f + exp(-gate_value))) * up_tile[element]);
}
}
kernel void matvec_f16(device const half* weights [[buffer(0)]], device const half* input [[buffer(1)]], device half* output [[buffer(2)]], constant MatvecParams& params [[buffer(3)]], uint group [[threadgroup_position_in_grid]], uint simd [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]]) {
uint row = group * 4 + simd;
if (row >= params.out_dim) return;
device const half4* weight4 = (device const half4*)(weights + row * params.in_dim);
device const half4* input4 = (device const half4*)input;
float sum = 0.0f;
for (uint i = lane; i < params.in_dim / 4; i += 32) {
sum += dot(float4(weight4[i]), float4(input4[i]));
}
sum = simd_sum(sum);
if (lane == 0) {
if (params.epi == kEpiAdd) sum += float(output[row]);
output[row] = half(sum);
}
}
kernel void matvec_gateup_swiglu(device const half* gate [[buffer(0)]], device const half* up [[buffer(1)]], device const half* input [[buffer(2)]], device half* output [[buffer(3)]], constant MatvecParams& params [[buffer(4)]], uint group [[threadgroup_position_in_grid]], uint simd [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]]) {
uint row = group * 4 + simd;
if (row >= params.out_dim) return;
device const half4* gate4 = (device const half4*)(gate + row * params.in_dim);
device const half4* up4 = (device const half4*)(up + row * params.in_dim);
device const half4* input4 = (device const half4*)input;
float gate_sum = 0.0f;
float up_sum = 0.0f;
for (uint i = lane; i < params.in_dim / 4; i += 32) {
float4 value = float4(input4[i]);
gate_sum += dot(float4(gate4[i]), value);
up_sum += dot(float4(up4[i]), value);
}
gate_sum = simd_sum(gate_sum);
up_sum = simd_sum(up_sum);
if (lane == 0) output[row] = half((gate_sum / (1.0f + exp(-gate_sum))) * up_sum);
}
kernel void rope_store_qkv(device const half* qkv [[buffer(0)]], device half* query [[buffer(1)]], device half* key_cache [[buffer(2)]], device half* value_cache [[buffer(3)]], constant Config& config [[buffer(4)]], constant uint& start [[buffer(5)]], uint2 gid [[thread_position_in_grid]]) {
uint half_dim = config.head_dim / 2;
uint index = gid.x;
uint token = gid.y;
if (index >= config.n_heads * half_dim) return;
uint head = index / half_dim;
uint pair = index % half_dim;
uint low = head * config.head_dim + pair;
uint high = low + half_dim;
uint position = start + token;
device const half* row = qkv + token * 3 * config.d_model;
float frequency = pow(10000.0f, -2.0f * float(pair) / float(config.head_dim));
float angle = float(position) * frequency;
float cosine = cos(angle);
float sine = sin(angle);
float query_low = float(row[low]);
float query_high = float(row[high]);
query[token * config.d_model + low] = half(query_low * cosine - query_high * sine);
query[token * config.d_model + high] = half(query_low * sine + query_high * cosine);
float key_low = float(row[config.d_model + low]);
float key_high = float(row[config.d_model + high]);
key_cache[position * config.d_model + low] = half(key_low * cosine - key_high * sine);
key_cache[position * config.d_model + high] = half(key_low * sine + key_high * cosine);
value_cache[position * config.d_model + low] = row[2 * config.d_model + low];
value_cache[position * config.d_model + high] = row[2 * config.d_model + high];
}
kernel void attention_rows(device const half* query [[buffer(0)]], device const half* key_cache [[buffer(1)]], device const half* value_cache [[buffer(2)]], device half* output [[buffer(3)]], constant Config& config [[buffer(4)]], constant uint& start [[buffer(5)]], uint2 group [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) {
threadgroup float scores[kMaxSeq];
uint head = group.x;
uint token = group.y;
uint head_dim = config.head_dim;
uint base = head * head_dim;
uint last = start + token;
float scale = rsqrt(float(head_dim));
device const half4* query4 = (device const half4*)(query + token * config.d_model + base);
float local_max = -INFINITY;
for (uint j = lane; j <= last; j += 32) {
device const half4* key4 = (device const half4*)(key_cache + j * config.d_model + base);
float score = 0.0f;
for (uint dim = 0; dim < head_dim / 4; ++dim) score += dot(float4(query4[dim]), float4(key4[dim]));
score *= scale;
scores[j] = score;
local_max = max(local_max, score);
}
float max_score = simd_max(local_max);
float local_sum = 0.0f;
for (uint j = lane; j <= last; j += 32) {
float value = exp(scores[j] - max_score);
scores[j] = value;
local_sum += value;
}
float sum = simd_sum(local_sum);
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint dim = lane; dim < head_dim; dim += 32) {
float acc = 0.0f;
for (uint j = 0; j <= last; ++j) acc += scores[j] * float(value_cache[j * config.d_model + base + dim]);
output[token * config.d_model + base + dim] = half(acc / sum);
}
}
kernel void argmax_half(device const half* logits [[buffer(0)]], device uint* tokens [[buffer(1)]], device uint* generated [[buffer(2)]], constant Config& config [[buffer(3)]], constant uint& slot [[buffer(4)]], uint lane [[thread_index_in_threadgroup]]) {
threadgroup float values[256];
threadgroup uint indexes[256];
float best = -INFINITY;
uint best_index = 0;
for (uint i = lane; i < config.vocab; i += 256) {
float value = float(logits[i]);
if (value > best) {
best = value;
best_index = i;
}
}
values[lane] = best;
indexes[lane] = best_index;
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint span = 128; span > 0; span >>= 1) {
if (lane < span && values[lane + span] > values[lane]) {
values[lane] = values[lane + span];
indexes[lane] = indexes[lane + span];
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
if (lane == 0) {
tokens[0] = indexes[0];
generated[slot] = indexes[0];
}
}
//
// inference_objc.m
// libo1.dev
//
// Created by gpt-6.1-xhigh on 2026-10-19.
//
/// Objective-C host that runs batched prefill and single-command-buffer decode over a KV-cache and
/// self-checks that the GEMM (prefill) and matvec (decode) paths produce matching logits.
///
/// Build: `clang -fobjc-arc -O2 -Wall -framework Foundation -framework Metal inference_objc.m -o infer`
/// Run: `./infer "your query"` (`inference.metal` must be in the current directory)
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#include <math.h>
typedef struct {
uint32_t d_model;
uint32_t n_heads;
uint32_t head_dim;
uint32_t ffn_dim;
uint32_t max_seq;
uint32_t vocab;
} config_t;
typedef struct {
uint32_t m;
uint32_t n;
uint32_t k;
uint32_t epi;
} gemm_params_t;
typedef struct {
uint32_t in_dim;
uint32_t out_dim;
uint32_t epi;
} matvec_params_t;
enum {
kLayers = 2,
kEpiStore = 0,
kEpiAdd = 1,
};
#pragma mark - Engine
@interface Engine : NSObject
- (instancetype)initWithDevice:(id<MTLDevice>)device;
- (void)prefill:(const uint8_t *)tokens count:(uint32_t)count;
- (void)prefillViaDecode:(const uint8_t *)tokens count:(uint32_t)count;
- (void)decode:(uint32_t)steps from:(uint32_t)position;
- (NSArray<NSNumber *> *)logitsSnapshot;
- (const uint32_t *)generated;
@property (nonatomic, readonly) config_t config;
@end
@implementation Engine {
id<MTLDevice> _device;
id<MTLCommandQueue> _queue;
config_t _config;
id<MTLComputePipelineState> _embed;
id<MTLComputePipelineState> _norm;
id<MTLComputePipelineState> _gemm;
id<MTLComputePipelineState> _gateUpGemm;
id<MTLComputePipelineState> _matvec;
id<MTLComputePipelineState> _gateUpMatvec;
id<MTLComputePipelineState> _rope;
id<MTLComputePipelineState> _attention;
id<MTLComputePipelineState> _argmax;
id<MTLBuffer> _embedding;
id<MTLBuffer> _finalNorm;
id<MTLBuffer> _hidden;
id<MTLBuffer> _normalized;
id<MTLBuffer> _normalizedLast;
id<MTLBuffer> _qkv;
id<MTLBuffer> _query;
id<MTLBuffer> _attentionOut;
id<MTLBuffer> _activation;
id<MTLBuffer> _logits;
id<MTLBuffer> _tokens;
id<MTLBuffer> _generated;
NSMutableArray<NSDictionary<NSString *, id<MTLBuffer>> *> *_layers;
NSMutableArray<id<MTLBuffer>> *_keyCaches;
NSMutableArray<id<MTLBuffer>> *_valueCaches;
}
- (config_t)config {
return _config;
}
- (instancetype)initWithDevice:(id<MTLDevice>)device {
self = [super init];
if (self) {
_device = device;
_queue = [device newCommandQueue];
_config = (config_t){
.d_model = 64,
.n_heads = 4,
.head_dim = 16,
.ffn_dim = 128,
.max_seq = 128,
.vocab = 256,
};
NSError *error = nil;
NSString *source = [NSString stringWithContentsOfFile:@"inference.metal" encoding:NSUTF8StringEncoding error:&error];
NSAssert(source, @"failed to read inference.metal: %@", error);
id<MTLLibrary> library = [device newLibraryWithSource:source options:nil error:&error];
NSAssert(library, @"metal compile error: %@", error);
_embed = [self pipeline:@"embed_rows" library:library];
_norm = [self pipeline:@"rmsnorm_rows" library:library];
_gemm = [self pipeline:@"gemm_f16" library:library];
_gateUpGemm = [self pipeline:@"gemm_gateup_swiglu" library:library];
_matvec = [self pipeline:@"matvec_f16" library:library];
_gateUpMatvec = [self pipeline:@"matvec_gateup_swiglu" library:library];
_rope = [self pipeline:@"rope_store_qkv" library:library];
_attention = [self pipeline:@"attention_rows" library:library];
_argmax = [self pipeline:@"argmax_half" library:library];
uint32_t modelDim = _config.d_model;
uint32_t ffnDim = _config.ffn_dim;
uint32_t vocab = _config.vocab;
uint32_t maxSeq = _config.max_seq;
srand48(7);
_embedding = [self halfRandom:vocab * modelDim scale:0.1f];
_finalNorm = [self halfOnes:modelDim];
_layers = [NSMutableArray array];
_keyCaches = [NSMutableArray array];
_valueCaches = [NSMutableArray array];
for (uint32_t layer = 0; layer < kLayers; ++layer) {
[_layers addObject:@{
@"attnNorm": [self halfOnes:modelDim],
@"wqkv": [self halfRandom:3 * modelDim * modelDim scale:0.1f],
@"wo": [self halfRandom:modelDim * modelDim scale:0.1f],
@"ffnNorm": [self halfOnes:modelDim],
@"wgate": [self halfRandom:ffnDim * modelDim scale:0.1f],
@"wup": [self halfRandom:ffnDim * modelDim scale:0.1f],
@"wdown": [self halfRandom:modelDim * ffnDim scale:0.1f],
}];
[_keyCaches addObject:[self halfZeros:maxSeq * modelDim]];
[_valueCaches addObject:[self halfZeros:maxSeq * modelDim]];
}
_hidden = [self halfZeros:maxSeq * modelDim];
_normalized = [self halfZeros:maxSeq * modelDim];
_normalizedLast = [self halfZeros:modelDim];
_qkv = [self halfZeros:maxSeq * 3 * modelDim];
_query = [self halfZeros:maxSeq * modelDim];
_attentionOut = [self halfZeros:maxSeq * modelDim];
_activation = [self halfZeros:maxSeq * ffnDim];
_logits = [self halfZeros:vocab];
_tokens = [self zeros:maxSeq * sizeof(uint32_t)];
_generated = [self zeros:maxSeq * sizeof(uint32_t)];
}
return self;
}
#pragma mark - Public
- (void)prefill:(const uint8_t *)tokens count:(uint32_t)count {
uint32_t *ids = _tokens.contents;
for (uint32_t i = 0; i < count; ++i) ids[i] = tokens[i];
[self run:^(id<MTLComputeCommandEncoder> encoder) {
[self encodeForward:encoder count:count start:0 slot:0];
}];
}
- (void)prefillViaDecode:(const uint8_t *)tokens count:(uint32_t)count {
for (uint32_t i = 0; i < count; ++i) {
((uint32_t *)_tokens.contents)[0] = tokens[i];
[self run:^(id<MTLComputeCommandEncoder> encoder) {
[self encodeForward:encoder count:1 start:i slot:0];
}];
}
}
- (void)decode:(uint32_t)steps from:(uint32_t)position {
[self run:^(id<MTLComputeCommandEncoder> encoder) {
for (uint32_t i = 0; i < steps; ++i) [self encodeForward:encoder count:1 start:position + i slot:i + 1];
}];
}
- (NSArray<NSNumber *> *)logitsSnapshot {
const _Float16 *values = _logits.contents;
NSMutableArray *logits = [NSMutableArray array];
for (uint32_t i = 0; i < _config.vocab; ++i) [logits addObject:@((float)values[i])];
return logits;
}
- (const uint32_t *)generated {
return _generated.contents;
}
#pragma mark - Private
- (id<MTLComputePipelineState>)pipeline:(NSString *)name library:(id<MTLLibrary>)library {
id<MTLFunction> function = [library newFunctionWithName:name];
NSError *error = nil;
id<MTLComputePipelineState> pipeline = function ? [_device newComputePipelineStateWithFunction:function error:&error] : nil;
NSAssert(pipeline, @"pipeline '%@' failed: %@", name, error);
return pipeline;
}
- (id<MTLBuffer>)zeros:(size_t)bytes {
id<MTLBuffer> buffer = [_device newBufferWithLength:bytes options:MTLResourceStorageModeShared];
memset(buffer.contents, 0, bytes);
return buffer;
}
- (id<MTLBuffer>)halfZeros:(size_t)count {
return [self zeros:count * sizeof(_Float16)];
}
- (id<MTLBuffer>)halfRandom:(size_t)count scale:(float)scale {
id<MTLBuffer> buffer = [self halfZeros:count];
_Float16 *values = buffer.contents;
for (size_t i = 0; i < count; ++i) values[i] = (_Float16)((drand48() * 2.0 - 1.0) * scale);
return buffer;
}
- (id<MTLBuffer>)halfOnes:(size_t)count {
id<MTLBuffer> buffer = [self halfZeros:count];
_Float16 *values = buffer.contents;
for (size_t i = 0; i < count; ++i) values[i] = (_Float16)1.0f;
return buffer;
}
- (void)run:(void (^)(id<MTLComputeCommandEncoder>))encode {
id<MTLCommandBuffer> commandBuffer = [_queue commandBuffer];
id<MTLComputeCommandEncoder> encoder = [commandBuffer computeCommandEncoder];
encode(encoder);
[encoder endEncoding];
[commandBuffer commit];
[commandBuffer waitUntilCompleted];
NSAssert(!commandBuffer.error, @"GPU error: %@", commandBuffer.error);
}
- (void)encodeNorm:(id<MTLComputeCommandEncoder>)encoder input:(id<MTLBuffer>)input offset:(NSUInteger)offset weight:(id<MTLBuffer>)weight output:(id<MTLBuffer>)output rows:(uint32_t)rows {
config_t config = _config;
[encoder setComputePipelineState:_norm];
[encoder setBuffer:input offset:offset atIndex:0];
[encoder setBuffer:weight offset:0 atIndex:1];
[encoder setBuffer:output offset:0 atIndex:2];
[encoder setBytes:&config length:sizeof(config) atIndex:3];
[encoder dispatchThreadgroups:MTLSizeMake(rows, 1, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
}
- (void)encodeLinear:(id<MTLComputeCommandEncoder>)encoder input:(id<MTLBuffer>)input weights:(id<MTLBuffer>)weights output:(id<MTLBuffer>)output count:(uint32_t)count columns:(uint32_t)columns inner:(uint32_t)inner add:(BOOL)add {
if (count > 1) {
gemm_params_t params = { (count + 7) & ~7u, columns, inner, add ? kEpiAdd : kEpiStore };
[encoder setComputePipelineState:_gemm];
[encoder setBuffer:input offset:0 atIndex:0];
[encoder setBuffer:weights offset:0 atIndex:1];
[encoder setBuffer:output offset:0 atIndex:2];
[encoder setBytes:¶ms length:sizeof(params) atIndex:3];
[encoder dispatchThreadgroups:MTLSizeMake(columns / 16, params.m / 8, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
} else {
matvec_params_t params = { inner, columns, add ? kEpiAdd : kEpiStore };
[encoder setComputePipelineState:_matvec];
[encoder setBuffer:weights offset:0 atIndex:0];
[encoder setBuffer:input offset:0 atIndex:1];
[encoder setBuffer:output offset:0 atIndex:2];
[encoder setBytes:¶ms length:sizeof(params) atIndex:3];
[encoder dispatchThreadgroups:MTLSizeMake((columns + 3) / 4, 1, 1) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
}
}
- (void)encodeGateUp:(id<MTLComputeCommandEncoder>)encoder input:(id<MTLBuffer>)input gate:(id<MTLBuffer>)gate up:(id<MTLBuffer>)up output:(id<MTLBuffer>)output count:(uint32_t)count {
uint32_t modelDim = _config.d_model;
uint32_t ffnDim = _config.ffn_dim;
if (count > 1) {
gemm_params_t params = { (count + 7) & ~7u, ffnDim, modelDim, kEpiStore };
[encoder setComputePipelineState:_gateUpGemm];
[encoder setBuffer:input offset:0 atIndex:0];
[encoder setBuffer:gate offset:0 atIndex:1];
[encoder setBuffer:up offset:0 atIndex:2];
[encoder setBuffer:output offset:0 atIndex:3];
[encoder setBytes:¶ms length:sizeof(params) atIndex:4];
[encoder dispatchThreadgroups:MTLSizeMake(ffnDim / 8, params.m / 8, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
} else {
matvec_params_t params = { modelDim, ffnDim, kEpiStore };
[encoder setComputePipelineState:_gateUpMatvec];
[encoder setBuffer:gate offset:0 atIndex:0];
[encoder setBuffer:up offset:0 atIndex:1];
[encoder setBuffer:input offset:0 atIndex:2];
[encoder setBuffer:output offset:0 atIndex:3];
[encoder setBytes:¶ms length:sizeof(params) atIndex:4];
[encoder dispatchThreadgroups:MTLSizeMake((ffnDim + 3) / 4, 1, 1) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
}
}
- (void)encodeForward:(id<MTLComputeCommandEncoder>)encoder count:(uint32_t)count start:(uint32_t)start slot:(uint32_t)slot {
config_t config = _config;
uint32_t modelDim = config.d_model;
uint32_t vocab = config.vocab;
[encoder setComputePipelineState:_embed];
[encoder setBuffer:_embedding offset:0 atIndex:0];
[encoder setBuffer:_tokens offset:0 atIndex:1];
[encoder setBuffer:_hidden offset:0 atIndex:2];
[encoder setBytes:&config length:sizeof(config) atIndex:3];
[encoder dispatchThreads:MTLSizeMake(modelDim, count, 1) threadsPerThreadgroup:MTLSizeMake(MIN(modelDim, 64), 1, 1)];
for (uint32_t layerIndex = 0; layerIndex < kLayers; ++layerIndex) {
NSDictionary<NSString *, id<MTLBuffer>> *layer = _layers[layerIndex];
[self encodeNorm:encoder input:_hidden offset:0 weight:layer[@"attnNorm"] output:_normalized rows:count];
[self encodeLinear:encoder input:_normalized weights:layer[@"wqkv"] output:_qkv count:count columns:3 * modelDim inner:modelDim add:NO];
[encoder setComputePipelineState:_rope];
[encoder setBuffer:_qkv offset:0 atIndex:0];
[encoder setBuffer:_query offset:0 atIndex:1];
[encoder setBuffer:_keyCaches[layerIndex] offset:0 atIndex:2];
[encoder setBuffer:_valueCaches[layerIndex] offset:0 atIndex:3];
[encoder setBytes:&config length:sizeof(config) atIndex:4];
[encoder setBytes:&start length:sizeof(start) atIndex:5];
[encoder dispatchThreads:MTLSizeMake(config.n_heads * config.head_dim / 2, count, 1) threadsPerThreadgroup:MTLSizeMake(MIN(config.n_heads * config.head_dim / 2, 32), 1, 1)];
[encoder setComputePipelineState:_attention];
[encoder setBuffer:_query offset:0 atIndex:0];
[encoder setBuffer:_keyCaches[layerIndex] offset:0 atIndex:1];
[encoder setBuffer:_valueCaches[layerIndex] offset:0 atIndex:2];
[encoder setBuffer:_attentionOut offset:0 atIndex:3];
[encoder setBytes:&config length:sizeof(config) atIndex:4];
[encoder setBytes:&start length:sizeof(start) atIndex:5];
[encoder dispatchThreadgroups:MTLSizeMake(config.n_heads, count, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
[self encodeLinear:encoder input:_attentionOut weights:layer[@"wo"] output:_hidden count:count columns:modelDim inner:modelDim add:YES];
[self encodeNorm:encoder input:_hidden offset:0 weight:layer[@"ffnNorm"] output:_normalized rows:count];
[self encodeGateUp:encoder input:_normalized gate:layer[@"wgate"] up:layer[@"wup"] output:_activation count:count];
[self encodeLinear:encoder input:_activation weights:layer[@"wdown"] output:_hidden count:count columns:modelDim inner:config.ffn_dim add:YES];
}
[self encodeNorm:encoder input:_hidden offset:(NSUInteger)(count - 1) * modelDim * sizeof(_Float16) weight:_finalNorm output:_normalizedLast rows:1];
[self encodeLinear:encoder input:_normalizedLast weights:_embedding output:_logits count:1 columns:vocab inner:modelDim add:NO];
[encoder setComputePipelineState:_argmax];
[encoder setBuffer:_logits offset:0 atIndex:0];
[encoder setBuffer:_tokens offset:0 atIndex:1];
[encoder setBuffer:_generated offset:0 atIndex:2];
[encoder setBytes:&config length:sizeof(config) atIndex:3];
[encoder setBytes:&slot length:sizeof(slot) atIndex:4];
[encoder dispatchThreadgroups:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];
}
@end
int main(int argc, const char *argv[]) {
@autoreleasepool {
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
NSAssert(device, @"metal device not supported");
NSAssert([device supportsFamily:MTLGPUFamilyApple7], @"Apple7+ GPU (M1 or newer) required");
NSLog(@"GPU: %@", device.name);
Engine *engine = [[Engine alloc] initWithDevice:device];
uint32_t maxSeq = engine.config.max_seq;
NSString *query = argc > 1 ? [NSString stringWithUTF8String:argv[1]] : @"Hello, optimized Metal!";
NSData *bytes = [query dataUsingEncoding:NSUTF8StringEncoding];
uint32_t count = (uint32_t)MIN(bytes.length, (NSUInteger)(maxSeq / 2));
const uint8_t *tokens = bytes.bytes;
[engine prefill:tokens count:count];
NSArray<NSNumber *> *batched = [engine logitsSnapshot];
uint32_t batchedToken = engine.generated[0];
[engine prefillViaDecode:tokens count:count];
NSArray<NSNumber *> *decoded = [engine logitsSnapshot];
float maxDifference = 0.0f;
for (NSUInteger i = 0; i < batched.count; ++i) maxDifference = fmaxf(maxDifference, fabsf(batched[i].floatValue - decoded[i].floatValue));
NSLog(@"self-check: max = %.5f, same argmax = %@ (expect small fp16 noise)", maxDifference, batchedToken == engine.generated[0] ? @"yes" : @"NO");
uint32_t steps = MIN(31u, maxSeq - count);
CFAbsoluteTime prefillStart = CFAbsoluteTimeGetCurrent();
[engine prefill:tokens count:count];
CFAbsoluteTime decodeStart = CFAbsoluteTimeGetCurrent();
[engine decode:steps from:count];
CFAbsoluteTime decodeEnd = CFAbsoluteTimeGetCurrent();
const uint32_t *generated = engine.generated;
NSMutableString *output = [NSMutableString string];
for (uint32_t i = 0; i <= steps; ++i) {
unichar character = (generated[i] >= 32 && generated[i] < 127) ? (unichar)generated[i] : '.';
[output appendFormat:@"%C", character];
}
NSLog(@"query: %@", query);
NSLog(@"output: %@", output);
NSLog(@"prefill: %u tokens in %.2f ms | decode: %u tokens in %.2f ms (%.0f tok/s)", count, (decodeStart - prefillStart) * 1e3, steps, (decodeEnd - decodeStart) * 1e3, steps / (decodeEnd - decodeStart));
}
return 0;
}
CC = clang
CFLAGS = -fobjc-arc -O2 -Wall
METAL = -framework Foundation -framework Metal
all: train infer
train: training_objc.m training.metal
$(CC) $(CFLAGS) $(METAL) training_objc.m -o train
infer: inference_objc.m inference.metal
$(CC) $(CFLAGS) $(METAL) inference_objc.m -o infer
clean:
rm -f train infer
.PHONY: all clean
//
// training.metal
// libo1.dev
//
// Created by gpt-6.1-xhigh on 2026-10-19.
//
/// Metal compute kernels for training a multilayer perceptron (MLP) on the GPU: forward pass,
/// backpropagation and stochastic gradient descent (SGD) using row-major fp32 weights and
/// activations.
#include <metal_stdlib>
using namespace metal;
struct Params {
uint in_dim;
uint out_dim;
uint batch;
uint activation;
float learning_rate;
};
constant uint kActTanh = 0;
constant uint kActSigmoid = 1;
static float activate(float z, uint activation) {
if (activation == kActTanh) return tanh(z);
if (activation == kActSigmoid) return 1.0f / (1.0f + exp(-z));
return z;
}
kernel void forward_layer(device const float* input [[buffer(0)]], device const float* weights [[buffer(1)]], device const float* bias [[buffer(2)]], device float* output [[buffer(3)]], constant Params& params [[buffer(4)]], uint2 gid [[thread_position_in_grid]]) {
uint neuron = gid.x;
uint sample = gid.y;
if (neuron >= params.out_dim || sample >= params.batch) return;
float z = bias[neuron];
for (uint i = 0; i < params.in_dim; ++i) {
z += weights[neuron * params.in_dim + i] * input[sample * params.in_dim + i];
}
output[sample * params.out_dim + neuron] = activate(z, params.activation);
}
kernel void output_delta(device const float* prediction [[buffer(0)]], device const float* target [[buffer(1)]], device float* delta [[buffer(2)]], constant Params& params [[buffer(3)]], uint2 gid [[thread_position_in_grid]]) {
uint neuron = gid.x;
uint sample = gid.y;
if (neuron >= params.out_dim || sample >= params.batch) return;
uint index = sample * params.out_dim + neuron;
delta[index] = (prediction[index] - target[index]) / float(params.batch);
}
kernel void hidden_delta(device const float* next_weights [[buffer(0)]], device const float* next_delta [[buffer(1)]], device const float* activation [[buffer(2)]], device float* delta [[buffer(3)]], constant Params& params [[buffer(4)]], constant Params& next_params [[buffer(5)]], uint2 gid [[thread_position_in_grid]]) {
uint hidden = gid.x;
uint sample = gid.y;
if (hidden >= params.out_dim || sample >= params.batch) return;
float sum = 0.0f;
for (uint neuron = 0; neuron < next_params.out_dim; ++neuron) {
sum += next_weights[neuron * next_params.in_dim + hidden] * next_delta[sample * next_params.out_dim + neuron];
}
float value = activation[sample * params.out_dim + hidden];
delta[sample * params.out_dim + hidden] = sum * (1.0f - value * value);
}
kernel void update_layer(device const float* input [[buffer(0)]], device const float* delta [[buffer(1)]], device float* weights [[buffer(2)]], device float* bias [[buffer(3)]], constant Params& params [[buffer(4)]], uint2 gid [[thread_position_in_grid]]) {
uint i = gid.x;
uint neuron = gid.y;
if (i > params.in_dim || neuron >= params.out_dim) return;
float gradient = 0.0f;
for (uint sample = 0; sample < params.batch; ++sample) {
float d = delta[sample * params.out_dim + neuron];
float x = (i == params.in_dim) ? 1.0f : input[sample * params.in_dim + i];
gradient += d * x;
}
if (i == params.in_dim) {
bias[neuron] -= params.learning_rate * gradient;
} else {
weights[neuron * params.in_dim + i] -= params.learning_rate * gradient;
}
}
//
// training_objc.m
// libo1.dev
//
// Created by gpt-6.1-xhigh on 2026-10-19.
//
/// Objective-C host that compiles `training.metal` at runtime, allocates shared-memory buffers and
/// encodes the training loop (500 epochs per command buffer) for a 2-8-1 MLP that learns XOR.
///
/// Build: `clang -fobjc-arc -O2 -Wall -framework Foundation -framework Metal training_objc.m -o train`
/// Run: `./train` (`training.metal` must be in the current directory)
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#include <math.h>
typedef struct {
uint32_t in_dim;
uint32_t out_dim;
uint32_t batch;
uint32_t activation;
float learning_rate;
} params_t;
enum {
kInput = 2,
kHidden = 8,
kOutput = 1,
kBatch = 4,
kEpochs = 10000,
kEpochsPerCommandBuffer = 500,
kActTanh = 0,
kActSigmoid = 1,
};
static const float kLearningRate = 0.8f;
static id<MTLComputePipelineState> make_pipeline(id<MTLDevice>, id<MTLLibrary>, NSString *);
static id<MTLBuffer> make_buffer(id<MTLDevice>, const float *, size_t);
static id<MTLBuffer> make_zero_buffer(id<MTLDevice>, size_t);
static void init_weights(float *, uint32_t, uint32_t);
static void dispatch_2d(id<MTLComputeCommandEncoder>, id<MTLComputePipelineState>, NSUInteger, NSUInteger);
int main(int argc, const char *argv[]) {
@autoreleasepool {
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
NSAssert(device, @"metal device not supported");
NSLog(@"GPU: %@", device.name);
id<MTLCommandQueue> queue = [device newCommandQueue];
NSError *error = nil;
NSString *source = [NSString stringWithContentsOfFile:@"training.metal" encoding:NSUTF8StringEncoding error:&error];
NSAssert(source, @"failed to read training.metal: %@", error);
id<MTLLibrary> library = [device newLibraryWithSource:source options:nil error:&error];
NSAssert(library, @"metal compile error: %@", error);
id<MTLComputePipelineState> forwardPipeline = make_pipeline(device, library, @"forward_layer");
id<MTLComputePipelineState> outputDeltaPipeline = make_pipeline(device, library, @"output_delta");
id<MTLComputePipelineState> hiddenDeltaPipeline = make_pipeline(device, library, @"hidden_delta");
id<MTLComputePipelineState> updatePipeline = make_pipeline(device, library, @"update_layer");
const float input[kBatch * kInput] = { 0, 0, 0, 1, 1, 0, 1, 1 };
const float target[kBatch * kOutput] = { 0, 1, 1, 0 };
float hidden_weight_values[kHidden * kInput];
float output_weight_values[kOutput * kHidden];
srand48(42);
init_weights(hidden_weight_values, kInput, kHidden);
init_weights(output_weight_values, kHidden, kOutput);
id<MTLBuffer> inputBuffer = make_buffer(device, input, kBatch * kInput);
id<MTLBuffer> targetBuffer = make_buffer(device, target, kBatch * kOutput);
id<MTLBuffer> hiddenWeights = make_buffer(device, hidden_weight_values, kHidden * kInput);
id<MTLBuffer> hiddenBias = make_zero_buffer(device, kHidden);
id<MTLBuffer> outputWeights = make_buffer(device, output_weight_values, kOutput * kHidden);
id<MTLBuffer> outputBias = make_zero_buffer(device, kOutput);
id<MTLBuffer> hiddenActivation = make_zero_buffer(device, kBatch * kHidden);
id<MTLBuffer> outputActivation = make_zero_buffer(device, kBatch * kOutput);
id<MTLBuffer> hiddenDelta = make_zero_buffer(device, kBatch * kHidden);
id<MTLBuffer> outputDelta = make_zero_buffer(device, kBatch * kOutput);
params_t hidden_params = {
.in_dim = kInput,
.out_dim = kHidden,
.batch = kBatch,
.activation = kActTanh,
.learning_rate = kLearningRate,
};
params_t output_params = {
.in_dim = kHidden,
.out_dim = kOutput,
.batch = kBatch,
.activation = kActSigmoid,
.learning_rate = kLearningRate,
};
for (int epoch = 0; epoch < kEpochs; epoch += kEpochsPerCommandBuffer) {
id<MTLCommandBuffer> commandBuffer = [queue commandBuffer];
id<MTLComputeCommandEncoder> encoder = [commandBuffer computeCommandEncoder];
for (int step = 0; step < kEpochsPerCommandBuffer; ++step) {
[encoder setBuffer:inputBuffer offset:0 atIndex:0];
[encoder setBuffer:hiddenWeights offset:0 atIndex:1];
[encoder setBuffer:hiddenBias offset:0 atIndex:2];
[encoder setBuffer:hiddenActivation offset:0 atIndex:3];
[encoder setBytes:&hidden_params length:sizeof(hidden_params) atIndex:4];
dispatch_2d(encoder, forwardPipeline, kHidden, kBatch);
[encoder setBuffer:hiddenActivation offset:0 atIndex:0];
[encoder setBuffer:outputWeights offset:0 atIndex:1];
[encoder setBuffer:outputBias offset:0 atIndex:2];
[encoder setBuffer:outputActivation offset:0 atIndex:3];
[encoder setBytes:&output_params length:sizeof(output_params) atIndex:4];
dispatch_2d(encoder, forwardPipeline, kOutput, kBatch);
[encoder setBuffer:outputActivation offset:0 atIndex:0];
[encoder setBuffer:targetBuffer offset:0 atIndex:1];
[encoder setBuffer:outputDelta offset:0 atIndex:2];
[encoder setBytes:&output_params length:sizeof(output_params) atIndex:3];
dispatch_2d(encoder, outputDeltaPipeline, kOutput, kBatch);
[encoder setBuffer:outputWeights offset:0 atIndex:0];
[encoder setBuffer:outputDelta offset:0 atIndex:1];
[encoder setBuffer:hiddenActivation offset:0 atIndex:2];
[encoder setBuffer:hiddenDelta offset:0 atIndex:3];
[encoder setBytes:&hidden_params length:sizeof(hidden_params) atIndex:4];
[encoder setBytes:&output_params length:sizeof(output_params) atIndex:5];
dispatch_2d(encoder, hiddenDeltaPipeline, kHidden, kBatch);
[encoder setBuffer:hiddenActivation offset:0 atIndex:0];
[encoder setBuffer:outputDelta offset:0 atIndex:1];
[encoder setBuffer:outputWeights offset:0 atIndex:2];
[encoder setBuffer:outputBias offset:0 atIndex:3];
[encoder setBytes:&output_params length:sizeof(output_params) atIndex:4];
dispatch_2d(encoder, updatePipeline, kHidden + 1, kOutput);
[encoder setBuffer:inputBuffer offset:0 atIndex:0];
[encoder setBuffer:hiddenDelta offset:0 atIndex:1];
[encoder setBuffer:hiddenWeights offset:0 atIndex:2];
[encoder setBuffer:hiddenBias offset:0 atIndex:3];
[encoder setBytes:&hidden_params length:sizeof(hidden_params) atIndex:4];
dispatch_2d(encoder, updatePipeline, kInput + 1, kHidden);
}
[encoder endEncoding];
[commandBuffer commit];
[commandBuffer waitUntilCompleted];
const float *prediction = (const float *)outputActivation.contents;
float loss = 0.0f;
for (uint32_t sample = 0; sample < kBatch; ++sample) {
float value = fminf(fmaxf(prediction[sample], 1e-7f), 1.0f - 1e-7f);
loss += -(target[sample] * logf(value) + (1.0f - target[sample]) * logf(1.0f - value));
}
NSLog(@"epoch %5d, loss %.6f", epoch + kEpochsPerCommandBuffer, loss / kBatch);
}
const float *prediction = (const float *)outputActivation.contents;
NSLog(@"final:");
for (uint32_t sample = 0; sample < kBatch; ++sample) NSLog(@" [%.0f, %.0f] -> %.4f (target %.0f)", input[sample * kInput], input[sample * kInput + 1], prediction[sample], target[sample]);
}
return 0;
}
static id<MTLComputePipelineState> make_pipeline(id<MTLDevice> device, id<MTLLibrary> library, NSString *name) {
NSError *error = nil;
id<MTLFunction> function = [library newFunctionWithName:name];
id<MTLComputePipelineState> pipeline = function ? [device newComputePipelineStateWithFunction:function error:&error] : nil;
NSAssert(pipeline, @"pipeline '%@' failed: %@", name, error);
return pipeline;
}
static id<MTLBuffer> make_buffer(id<MTLDevice> device, const float *data, size_t count) {
return [device newBufferWithBytes:data length:count * sizeof(float) options:MTLResourceStorageModeShared];
}
static id<MTLBuffer> make_zero_buffer(id<MTLDevice> device, size_t count) {
id<MTLBuffer> buffer = [device newBufferWithLength:count * sizeof(float) options:MTLResourceStorageModeShared];
memset(buffer.contents, 0, count * sizeof(float));
return buffer;
}
static void init_weights(float *weights, uint32_t in_dim, uint32_t out_dim) {
float limit = sqrtf(6.0f / (float)(in_dim + out_dim));
for (uint32_t i = 0; i < in_dim * out_dim; ++i) weights[i] = (float)(drand48() * 2.0 - 1.0) * limit;
}
static void dispatch_2d(id<MTLComputeCommandEncoder> encoder, id<MTLComputePipelineState> pipeline, NSUInteger width, NSUInteger height) {
NSUInteger threadWidth = MIN(width, 8);
NSUInteger threadHeight = MIN(height, 8);
[encoder setComputePipelineState:pipeline];
[encoder dispatchThreads:MTLSizeMake(width, height, 1) threadsPerThreadgroup:MTLSizeMake(threadWidth, threadHeight, 1)];
}
//
// inference.metal
// libo1.dev
//
// Created by gpt-6.1-xhigh on 2026-10-19.
//
/// Metal compute kernels for a LLaMA-style transformer decoder: fp16 storage with fp32 accumulation,
/// `simdgroup_matrix` GEMMs for prefill, vectorized matvec for decode, and fused QKV, RoPE, KV-
/// cache and SwiGLU ops. Requires Apple silicon.
#include <metal_stdlib>
#include <metal_simdgroup_matrix>
using namespace metal;
struct Config {
uint d_model;
uint n_heads;
uint head_dim;
uint ffn_dim;
uint max_seq;
uint vocab;
};
struct GemmParams {
uint m;
uint n;
uint k;
uint epi;
};
struct MatvecParams {
uint in_dim;
uint out_dim;
uint epi;
};
constant uint kMaxSeq = 128;
constant uint kEpiAdd = 1;
kernel void embed_rows(device const half* table [[buffer(0)]], device const uint* tokens [[buffer(1)]], device half* output [[buffer(2)]], constant Config& config [[buffer(3)]], uint2 gid [[thread_position_in_grid]]) {
uint dim = gid.x;
uint token = gid.y;
if (dim >= config.d_model) return;
output[token * config.d_model + dim] = table[tokens[token] * config.d_model + dim];
}
kernel void rmsnorm_rows(device const half* input [[buffer(0)]], device const half* weight [[buffer(1)]], device half* output [[buffer(2)]], constant Config& config [[buffer(3)]], uint row [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) {
device const half* row_input = input + row * config.d_model;
float sum_squares = 0.0f;
for (uint i = lane; i < config.d_model; i += 32) {
float value = float(row_input[i]);
sum_squares += value * value;
}
sum_squares = simd_sum(sum_squares);
float scale = rsqrt(sum_squares / float(config.d_model) + 1e-5f);
for (uint i = lane; i < config.d_model; i += 32) {
output[row * config.d_model + i] = half(float(row_input[i]) * scale * float(weight[i]));
}
}
kernel void gemm_f16(device const half* input [[buffer(0)]], device const half* weights [[buffer(1)]], device half* output [[buffer(2)]], constant GemmParams& params [[buffer(3)]], uint2 group [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) {
threadgroup float tile[8 * 16];
uint column0 = group.x * 16;
uint row0 = group.y * 8;
simdgroup_float8x8 left = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
simdgroup_float8x8 right = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
for (uint block = 0; block < params.k; block += 8) {
simdgroup_half8x8 a;
simdgroup_half8x8 b0;
simdgroup_half8x8 b1;
simdgroup_load(a, input + row0 * params.k + block, params.k);
simdgroup_load(b0, weights + column0 * params.k + block, params.k, ulong2(0, 0), true);
simdgroup_load(b1, weights + (column0 + 8) * params.k + block, params.k, ulong2(0, 0), true);
simdgroup_multiply_accumulate(left, a, b0, left);
simdgroup_multiply_accumulate(right, a, b1, right);
}
simdgroup_store(left, tile, 16);
simdgroup_store(right, tile + 8, 16);
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint element = lane; element < 8 * 16; element += 32) {
uint row = element / 16;
uint column = element % 16;
uint index = (row0 + row) * params.n + column0 + column;
float value = tile[element];
if (params.epi == kEpiAdd) value += float(output[index]);
output[index] = half(value);
}
}
kernel void gemm_gateup_swiglu(device const half* input [[buffer(0)]], device const half* gate [[buffer(1)]], device const half* up [[buffer(2)]], device half* output [[buffer(3)]], constant GemmParams& params [[buffer(4)]], uint2 group [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) {
threadgroup float gate_tile[64];
threadgroup float up_tile[64];
uint column0 = group.x * 8;
uint row0 = group.y * 8;
simdgroup_float8x8 gate_acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
simdgroup_float8x8 up_acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
for (uint block = 0; block < params.k; block += 8) {
simdgroup_half8x8 a;
simdgroup_half8x8 gate_block;
simdgroup_half8x8 up_block;
simdgroup_load(a, input + row0 * params.k + block, params.k);
simdgroup_load(gate_block, gate + column0 * params.k + block, params.k, ulong2(0, 0), true);
simdgroup_load(up_block, up + column0 * params.k + block, params.k, ulong2(0, 0), true);
simdgroup_multiply_accumulate(gate_acc, a, gate_block, gate_acc);
simdgroup_multiply_accumulate(up_acc, a, up_block, up_acc);
}
simdgroup_store(gate_acc, gate_tile, 8);
simdgroup_store(up_acc, up_tile, 8);
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint element = lane; element < 64; element += 32) {
uint row = element / 8;
uint column = element % 8;
float gate_value = gate_tile[element];
output[(row0 + row) * params.n + column0 + column] = half((gate_value / (1.0f + exp(-gate_value))) * up_tile[element]);
}
}
kernel void matvec_f16(device const half* weights [[buffer(0)]], device const half* input [[buffer(1)]], device half* output [[buffer(2)]], constant MatvecParams& params [[buffer(3)]], uint group [[threadgroup_position_in_grid]], uint simd [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]]) {
uint row = group * 4 + simd;
if (row >= params.out_dim) return;
device const half4* weight4 = (device const half4*)(weights + row * params.in_dim);
device const half4* input4 = (device const half4*)input;
float sum = 0.0f;
for (uint i = lane; i < params.in_dim / 4; i += 32) {
sum += dot(float4(weight4[i]), float4(input4[i]));
}
sum = simd_sum(sum);
if (lane == 0) {
if (params.epi == kEpiAdd) sum += float(output[row]);
output[row] = half(sum);
}
}
kernel void matvec_gateup_swiglu(device const half* gate [[buffer(0)]], device const half* up [[buffer(1)]], device const half* input [[buffer(2)]], device half* output [[buffer(3)]], constant MatvecParams& params [[buffer(4)]], uint group [[threadgroup_position_in_grid]], uint simd [[simdgroup_index_in_threadgroup]], uint lane [[thread_index_in_simdgroup]]) {
uint row = group * 4 + simd;
if (row >= params.out_dim) return;
device const half4* gate4 = (device const half4*)(gate + row * params.in_dim);
device const half4* up4 = (device const half4*)(up + row * params.in_dim);
device const half4* input4 = (device const half4*)input;
float gate_sum = 0.0f;
float up_sum = 0.0f;
for (uint i = lane; i < params.in_dim / 4; i += 32) {
float4 value = float4(input4[i]);
gate_sum += dot(float4(gate4[i]), value);
up_sum += dot(float4(up4[i]), value);
}
gate_sum = simd_sum(gate_sum);
up_sum = simd_sum(up_sum);
if (lane == 0) output[row] = half((gate_sum / (1.0f + exp(-gate_sum))) * up_sum);
}
kernel void rope_store_qkv(device const half* qkv [[buffer(0)]], device half* query [[buffer(1)]], device half* key_cache [[buffer(2)]], device half* value_cache [[buffer(3)]], constant Config& config [[buffer(4)]], constant uint& start [[buffer(5)]], uint2 gid [[thread_position_in_grid]]) {
uint half_dim = config.head_dim / 2;
uint index = gid.x;
uint token = gid.y;
if (index >= config.n_heads * half_dim) return;
uint head = index / half_dim;
uint pair = index % half_dim;
uint low = head * config.head_dim + pair;
uint high = low + half_dim;
uint position = start + token;
device const half* row = qkv + token * 3 * config.d_model;
float frequency = pow(10000.0f, -2.0f * float(pair) / float(config.head_dim));
float angle = float(position) * frequency;
float cosine = cos(angle);
float sine = sin(angle);
float query_low = float(row[low]);
float query_high = float(row[high]);
query[token * config.d_model + low] = half(query_low * cosine - query_high * sine);
query[token * config.d_model + high] = half(query_low * sine + query_high * cosine);
float key_low = float(row[config.d_model + low]);
float key_high = float(row[config.d_model + high]);
key_cache[position * config.d_model + low] = half(key_low * cosine - key_high * sine);
key_cache[position * config.d_model + high] = half(key_low * sine + key_high * cosine);
value_cache[position * config.d_model + low] = row[2 * config.d_model + low];
value_cache[position * config.d_model + high] = row[2 * config.d_model + high];
}
kernel void attention_rows(device const half* query [[buffer(0)]], device const half* key_cache [[buffer(1)]], device const half* value_cache [[buffer(2)]], device half* output [[buffer(3)]], constant Config& config [[buffer(4)]], constant uint& start [[buffer(5)]], uint2 group [[threadgroup_position_in_grid]], uint lane [[thread_index_in_threadgroup]]) {
threadgroup float scores[kMaxSeq];
uint head = group.x;
uint token = group.y;
uint head_dim = config.head_dim;
uint base = head * head_dim;
uint last = start + token;
float scale = rsqrt(float(head_dim));
device const half4* query4 = (device const half4*)(query + token * config.d_model + base);
float local_max = -INFINITY;
for (uint j = lane; j <= last; j += 32) {
device const half4* key4 = (device const half4*)(key_cache + j * config.d_model + base);
float score = 0.0f;
for (uint dim = 0; dim < head_dim / 4; ++dim) score += dot(float4(query4[dim]), float4(key4[dim]));
score *= scale;
scores[j] = score;
local_max = max(local_max, score);
}
float max_score = simd_max(local_max);
float local_sum = 0.0f;
for (uint j = lane; j <= last; j += 32) {
float value = exp(scores[j] - max_score);
scores[j] = value;
local_sum += value;
}
float sum = simd_sum(local_sum);
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint dim = lane; dim < head_dim; dim += 32) {
float acc = 0.0f;
for (uint j = 0; j <= last; ++j) acc += scores[j] * float(value_cache[j * config.d_model + base + dim]);
output[token * config.d_model + base + dim] = half(acc / sum);
}
}
kernel void argmax_half(device const half* logits [[buffer(0)]], device uint* tokens [[buffer(1)]], device uint* generated [[buffer(2)]], constant Config& config [[buffer(3)]], constant uint& slot [[buffer(4)]], uint lane [[thread_index_in_threadgroup]]) {
threadgroup float values[256];
threadgroup uint indexes[256];
float best = -INFINITY;
uint best_index = 0;
for (uint i = lane; i < config.vocab; i += 256) {
float value = float(logits[i]);
if (value > best) {
best = value;
best_index = i;
}
}
values[lane] = best;
indexes[lane] = best_index;
threadgroup_barrier(mem_flags::mem_threadgroup);
for (uint span = 128; span > 0; span >>= 1) {
if (lane < span && values[lane + span] > values[lane]) {
values[lane] = values[lane + span];
indexes[lane] = indexes[lane + span];
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
if (lane == 0) {
tokens[0] = indexes[0];
generated[slot] = indexes[0];
}
}
//
// inference_objc.m
// libo1.dev
//
// Created by gpt-6.1-xhigh on 2026-10-19.
//
/// Objective-C host that runs batched prefill and single-command-buffer decode over a KV-cache and
/// self-checks that the GEMM (prefill) and matvec (decode) paths produce matching logits.
///
/// Build: `clang -fobjc-arc -O2 -Wall -framework Foundation -framework Metal inference_objc.m -o infer`
/// Run: `./infer "your query"` (`inference.metal` must be in the current directory)
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#include <math.h>
typedef struct {
uint32_t d_model;
uint32_t n_heads;
uint32_t head_dim;
uint32_t ffn_dim;
uint32_t max_seq;
uint32_t vocab;
} config_t;
typedef struct {
uint32_t m;
uint32_t n;
uint32_t k;
uint32_t epi;
} gemm_params_t;
typedef struct {
uint32_t in_dim;
uint32_t out_dim;
uint32_t epi;
} matvec_params_t;
enum {
kLayers = 2,
kEpiStore = 0,
kEpiAdd = 1,
};
#pragma mark - Engine
@interface Engine : NSObject
- (instancetype)initWithDevice:(id<MTLDevice>)device;
- (void)prefill:(const uint8_t *)tokens count:(uint32_t)count;
- (void)prefillViaDecode:(const uint8_t *)tokens count:(uint32_t)count;
- (void)decode:(uint32_t)steps from:(uint32_t)position;
- (NSArray<NSNumber *> *)logitsSnapshot;
- (const uint32_t *)generated;
@property (nonatomic, readonly) config_t config;
@end
@implementation Engine {
id<MTLDevice> _device;
id<MTLCommandQueue> _queue;
config_t _config;
id<MTLComputePipelineState> _embed;
id<MTLComputePipelineState> _norm;
id<MTLComputePipelineState> _gemm;
id<MTLComputePipelineState> _gateUpGemm;
id<MTLComputePipelineState> _matvec;
id<MTLComputePipelineState> _gateUpMatvec;
id<MTLComputePipelineState> _rope;
id<MTLComputePipelineState> _attention;
id<MTLComputePipelineState> _argmax;
id<MTLBuffer> _embedding;
id<MTLBuffer> _finalNorm;
id<MTLBuffer> _hidden;
id<MTLBuffer> _normalized;
id<MTLBuffer> _normalizedLast;
id<MTLBuffer> _qkv;
id<MTLBuffer> _query;
id<MTLBuffer> _attentionOut;
id<MTLBuffer> _activation;
id<MTLBuffer> _logits;
id<MTLBuffer> _tokens;
id<MTLBuffer> _generated;
NSMutableArray<NSDictionary<NSString *, id<MTLBuffer>> *> *_layers;
NSMutableArray<id<MTLBuffer>> *_keyCaches;
NSMutableArray<id<MTLBuffer>> *_valueCaches;
}
- (config_t)config {
return _config;
}
- (instancetype)initWithDevice:(id<MTLDevice>)device {
self = [super init];
if (self) {
_device = device;
_queue = [device newCommandQueue];
_config = (config_t){
.d_model = 64,
.n_heads = 4,
.head_dim = 16,
.ffn_dim = 128,
.max_seq = 128,
.vocab = 256,
};
NSError *error = nil;
NSString *source = [NSString stringWithContentsOfFile:@"inference.metal" encoding:NSUTF8StringEncoding error:&error];
NSAssert(source, @"failed to read inference.metal: %@", error);
id<MTLLibrary> library = [device newLibraryWithSource:source options:nil error:&error];
NSAssert(library, @"metal compile error: %@", error);
_embed = [self pipeline:@"embed_rows" library:library];
_norm = [self pipeline:@"rmsnorm_rows" library:library];
_gemm = [self pipeline:@"gemm_f16" library:library];
_gateUpGemm = [self pipeline:@"gemm_gateup_swiglu" library:library];
_matvec = [self pipeline:@"matvec_f16" library:library];
_gateUpMatvec = [self pipeline:@"matvec_gateup_swiglu" library:library];
_rope = [self pipeline:@"rope_store_qkv" library:library];
_attention = [self pipeline:@"attention_rows" library:library];
_argmax = [self pipeline:@"argmax_half" library:library];
uint32_t modelDim = _config.d_model;
uint32_t ffnDim = _config.ffn_dim;
uint32_t vocab = _config.vocab;
uint32_t maxSeq = _config.max_seq;
srand48(7);
_embedding = [self halfRandom:vocab * modelDim scale:0.1f];
_finalNorm = [self halfOnes:modelDim];
_layers = [NSMutableArray array];
_keyCaches = [NSMutableArray array];
_valueCaches = [NSMutableArray array];
for (uint32_t layer = 0; layer < kLayers; ++layer) {
[_layers addObject:@{
@"attnNorm": [self halfOnes:modelDim],
@"wqkv": [self halfRandom:3 * modelDim * modelDim scale:0.1f],
@"wo": [self halfRandom:modelDim * modelDim scale:0.1f],
@"ffnNorm": [self halfOnes:modelDim],
@"wgate": [self halfRandom:ffnDim * modelDim scale:0.1f],
@"wup": [self halfRandom:ffnDim * modelDim scale:0.1f],
@"wdown": [self halfRandom:modelDim * ffnDim scale:0.1f],
}];
[_keyCaches addObject:[self halfZeros:maxSeq * modelDim]];
[_valueCaches addObject:[self halfZeros:maxSeq * modelDim]];
}
_hidden = [self halfZeros:maxSeq * modelDim];
_normalized = [self halfZeros:maxSeq * modelDim];
_normalizedLast = [self halfZeros:modelDim];
_qkv = [self halfZeros:maxSeq * 3 * modelDim];
_query = [self halfZeros:maxSeq * modelDim];
_attentionOut = [self halfZeros:maxSeq * modelDim];
_activation = [self halfZeros:maxSeq * ffnDim];
_logits = [self halfZeros:vocab];
_tokens = [self zeros:maxSeq * sizeof(uint32_t)];
_generated = [self zeros:maxSeq * sizeof(uint32_t)];
}
return self;
}
#pragma mark - Public
- (void)prefill:(const uint8_t *)tokens count:(uint32_t)count {
uint32_t *ids = _tokens.contents;
for (uint32_t i = 0; i < count; ++i) ids[i] = tokens[i];
[self run:^(id<MTLComputeCommandEncoder> encoder) {
[self encodeForward:encoder count:count start:0 slot:0];
}];
}
- (void)prefillViaDecode:(const uint8_t *)tokens count:(uint32_t)count {
for (uint32_t i = 0; i < count; ++i) {
((uint32_t *)_tokens.contents)[0] = tokens[i];
[self run:^(id<MTLComputeCommandEncoder> encoder) {
[self encodeForward:encoder count:1 start:i slot:0];
}];
}
}
- (void)decode:(uint32_t)steps from:(uint32_t)position {
[self run:^(id<MTLComputeCommandEncoder> encoder) {
for (uint32_t i = 0; i < steps; ++i) [self encodeForward:encoder count:1 start:position + i slot:i + 1];
}];
}
- (NSArray<NSNumber *> *)logitsSnapshot {
const _Float16 *values = _logits.contents;
NSMutableArray *logits = [NSMutableArray array];
for (uint32_t i = 0; i < _config.vocab; ++i) [logits addObject:@((float)values[i])];
return logits;
}
- (const uint32_t *)generated {
return _generated.contents;
}
#pragma mark - Private
- (id<MTLComputePipelineState>)pipeline:(NSString *)name library:(id<MTLLibrary>)library {
id<MTLFunction> function = [library newFunctionWithName:name];
NSError *error = nil;
id<MTLComputePipelineState> pipeline = function ? [_device newComputePipelineStateWithFunction:function error:&error] : nil;
NSAssert(pipeline, @"pipeline '%@' failed: %@", name, error);
return pipeline;
}
- (id<MTLBuffer>)zeros:(size_t)bytes {
id<MTLBuffer> buffer = [_device newBufferWithLength:bytes options:MTLResourceStorageModeShared];
memset(buffer.contents, 0, bytes);
return buffer;
}
- (id<MTLBuffer>)halfZeros:(size_t)count {
return [self zeros:count * sizeof(_Float16)];
}
- (id<MTLBuffer>)halfRandom:(size_t)count scale:(float)scale {
id<MTLBuffer> buffer = [self halfZeros:count];
_Float16 *values = buffer.contents;
for (size_t i = 0; i < count; ++i) values[i] = (_Float16)((drand48() * 2.0 - 1.0) * scale);
return buffer;
}
- (id<MTLBuffer>)halfOnes:(size_t)count {
id<MTLBuffer> buffer = [self halfZeros:count];
_Float16 *values = buffer.contents;
for (size_t i = 0; i < count; ++i) values[i] = (_Float16)1.0f;
return buffer;
}
- (void)run:(void (^)(id<MTLComputeCommandEncoder>))encode {
id<MTLCommandBuffer> commandBuffer = [_queue commandBuffer];
id<MTLComputeCommandEncoder> encoder = [commandBuffer computeCommandEncoder];
encode(encoder);
[encoder endEncoding];
[commandBuffer commit];
[commandBuffer waitUntilCompleted];
NSAssert(!commandBuffer.error, @"GPU error: %@", commandBuffer.error);
}
- (void)encodeNorm:(id<MTLComputeCommandEncoder>)encoder input:(id<MTLBuffer>)input offset:(NSUInteger)offset weight:(id<MTLBuffer>)weight output:(id<MTLBuffer>)output rows:(uint32_t)rows {
config_t config = _config;
[encoder setComputePipelineState:_norm];
[encoder setBuffer:input offset:offset atIndex:0];
[encoder setBuffer:weight offset:0 atIndex:1];
[encoder setBuffer:output offset:0 atIndex:2];
[encoder setBytes:&config length:sizeof(config) atIndex:3];
[encoder dispatchThreadgroups:MTLSizeMake(rows, 1, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
}
- (void)encodeLinear:(id<MTLComputeCommandEncoder>)encoder input:(id<MTLBuffer>)input weights:(id<MTLBuffer>)weights output:(id<MTLBuffer>)output count:(uint32_t)count columns:(uint32_t)columns inner:(uint32_t)inner add:(BOOL)add {
if (count > 1) {
gemm_params_t params = { (count + 7) & ~7u, columns, inner, add ? kEpiAdd : kEpiStore };
[encoder setComputePipelineState:_gemm];
[encoder setBuffer:input offset:0 atIndex:0];
[encoder setBuffer:weights offset:0 atIndex:1];
[encoder setBuffer:output offset:0 atIndex:2];
[encoder setBytes:¶ms length:sizeof(params) atIndex:3];
[encoder dispatchThreadgroups:MTLSizeMake(columns / 16, params.m / 8, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
} else {
matvec_params_t params = { inner, columns, add ? kEpiAdd : kEpiStore };
[encoder setComputePipelineState:_matvec];
[encoder setBuffer:weights offset:0 atIndex:0];
[encoder setBuffer:input offset:0 atIndex:1];
[encoder setBuffer:output offset:0 atIndex:2];
[encoder setBytes:¶ms length:sizeof(params) atIndex:3];
[encoder dispatchThreadgroups:MTLSizeMake((columns + 3) / 4, 1, 1) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
}
}
- (void)encodeGateUp:(id<MTLComputeCommandEncoder>)encoder input:(id<MTLBuffer>)input gate:(id<MTLBuffer>)gate up:(id<MTLBuffer>)up output:(id<MTLBuffer>)output count:(uint32_t)count {
uint32_t modelDim = _config.d_model;
uint32_t ffnDim = _config.ffn_dim;
if (count > 1) {
gemm_params_t params = { (count + 7) & ~7u, ffnDim, modelDim, kEpiStore };
[encoder setComputePipelineState:_gateUpGemm];
[encoder setBuffer:input offset:0 atIndex:0];
[encoder setBuffer:gate offset:0 atIndex:1];
[encoder setBuffer:up offset:0 atIndex:2];
[encoder setBuffer:output offset:0 atIndex:3];
[encoder setBytes:¶ms length:sizeof(params) atIndex:4];
[encoder dispatchThreadgroups:MTLSizeMake(ffnDim / 8, params.m / 8, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
} else {
matvec_params_t params = { modelDim, ffnDim, kEpiStore };
[encoder setComputePipelineState:_gateUpMatvec];
[encoder setBuffer:gate offset:0 atIndex:0];
[encoder setBuffer:up offset:0 atIndex:1];
[encoder setBuffer:input offset:0 atIndex:2];
[encoder setBuffer:output offset:0 atIndex:3];
[encoder setBytes:¶ms length:sizeof(params) atIndex:4];
[encoder dispatchThreadgroups:MTLSizeMake((ffnDim + 3) / 4, 1, 1) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
}
}
- (void)encodeForward:(id<MTLComputeCommandEncoder>)encoder count:(uint32_t)count start:(uint32_t)start slot:(uint32_t)slot {
config_t config = _config;
uint32_t modelDim = config.d_model;
uint32_t vocab = config.vocab;
[encoder setComputePipelineState:_embed];
[encoder setBuffer:_embedding offset:0 atIndex:0];
[encoder setBuffer:_tokens offset:0 atIndex:1];
[encoder setBuffer:_hidden offset:0 atIndex:2];
[encoder setBytes:&config length:sizeof(config) atIndex:3];
[encoder dispatchThreads:MTLSizeMake(modelDim, count, 1) threadsPerThreadgroup:MTLSizeMake(MIN(modelDim, 64), 1, 1)];
for (uint32_t layerIndex = 0; layerIndex < kLayers; ++layerIndex) {
NSDictionary<NSString *, id<MTLBuffer>> *layer = _layers[layerIndex];
[self encodeNorm:encoder input:_hidden offset:0 weight:layer[@"attnNorm"] output:_normalized rows:count];
[self encodeLinear:encoder input:_normalized weights:layer[@"wqkv"] output:_qkv count:count columns:3 * modelDim inner:modelDim add:NO];
[encoder setComputePipelineState:_rope];
[encoder setBuffer:_qkv offset:0 atIndex:0];
[encoder setBuffer:_query offset:0 atIndex:1];
[encoder setBuffer:_keyCaches[layerIndex] offset:0 atIndex:2];
[encoder setBuffer:_valueCaches[layerIndex] offset:0 atIndex:3];
[encoder setBytes:&config length:sizeof(config) atIndex:4];
[encoder setBytes:&start length:sizeof(start) atIndex:5];
[encoder dispatchThreads:MTLSizeMake(config.n_heads * config.head_dim / 2, count, 1) threadsPerThreadgroup:MTLSizeMake(MIN(config.n_heads * config.head_dim / 2, 32), 1, 1)];
[encoder setComputePipelineState:_attention];
[encoder setBuffer:_query offset:0 atIndex:0];
[encoder setBuffer:_keyCaches[layerIndex] offset:0 atIndex:1];
[encoder setBuffer:_valueCaches[layerIndex] offset:0 atIndex:2];
[encoder setBuffer:_attentionOut offset:0 atIndex:3];
[encoder setBytes:&config length:sizeof(config) atIndex:4];
[encoder setBytes:&start length:sizeof(start) atIndex:5];
[encoder dispatchThreadgroups:MTLSizeMake(config.n_heads, count, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
[self encodeLinear:encoder input:_attentionOut weights:layer[@"wo"] output:_hidden count:count columns:modelDim inner:modelDim add:YES];
[self encodeNorm:encoder input:_hidden offset:0 weight:layer[@"ffnNorm"] output:_normalized rows:count];
[self encodeGateUp:encoder input:_normalized gate:layer[@"wgate"] up:layer[@"wup"] output:_activation count:count];
[self encodeLinear:encoder input:_activation weights:layer[@"wdown"] output:_hidden count:count columns:modelDim inner:config.ffn_dim add:YES];
}
[self encodeNorm:encoder input:_hidden offset:(NSUInteger)(count - 1) * modelDim * sizeof(_Float16) weight:_finalNorm output:_normalizedLast rows:1];
[self encodeLinear:encoder input:_normalizedLast weights:_embedding output:_logits count:1 columns:vocab inner:modelDim add:NO];
[encoder setComputePipelineState:_argmax];
[encoder setBuffer:_logits offset:0 atIndex:0];
[encoder setBuffer:_tokens offset:0 atIndex:1];
[encoder setBuffer:_generated offset:0 atIndex:2];
[encoder setBytes:&config length:sizeof(config) atIndex:3];
[encoder setBytes:&slot length:sizeof(slot) atIndex:4];
[encoder dispatchThreadgroups:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];
}
@end
int main(int argc, const char *argv[]) {
@autoreleasepool {
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
NSAssert(device, @"metal device not supported");
NSAssert([device supportsFamily:MTLGPUFamilyApple7], @"Apple7+ GPU (M1 or newer) required");
NSLog(@"GPU: %@", device.name);
Engine *engine = [[Engine alloc] initWithDevice:device];
uint32_t maxSeq = engine.config.max_seq;
NSString *query = argc > 1 ? [NSString stringWithUTF8String:argv[1]] : @"Hello, optimized Metal!";
NSData *bytes = [query dataUsingEncoding:NSUTF8StringEncoding];
uint32_t count = (uint32_t)MIN(bytes.length, (NSUInteger)(maxSeq / 2));
const uint8_t *tokens = bytes.bytes;
[engine prefill:tokens count:count];
NSArray<NSNumber *> *batched = [engine logitsSnapshot];
uint32_t batchedToken = engine.generated[0];
[engine prefillViaDecode:tokens count:count];
NSArray<NSNumber *> *decoded = [engine logitsSnapshot];
float maxDifference = 0.0f;
for (NSUInteger i = 0; i < batched.count; ++i) maxDifference = fmaxf(maxDifference, fabsf(batched[i].floatValue - decoded[i].floatValue));
NSLog(@"self-check: max = %.5f, same argmax = %@ (expect small fp16 noise)", maxDifference, batchedToken == engine.generated[0] ? @"yes" : @"NO");
uint32_t steps = MIN(31u, maxSeq - count);
CFAbsoluteTime prefillStart = CFAbsoluteTimeGetCurrent();
[engine prefill:tokens count:count];
CFAbsoluteTime decodeStart = CFAbsoluteTimeGetCurrent();
[engine decode:steps from:count];
CFAbsoluteTime decodeEnd = CFAbsoluteTimeGetCurrent();
const uint32_t *generated = engine.generated;
NSMutableString *output = [NSMutableString string];
for (uint32_t i = 0; i <= steps; ++i) {
unichar character = (generated[i] >= 32 && generated[i] < 127) ? (unichar)generated[i] : '.';
[output appendFormat:@"%C", character];
}
NSLog(@"query: %@", query);
NSLog(@"output: %@", output);
NSLog(@"prefill: %u tokens in %.2f ms | decode: %u tokens in %.2f ms (%.0f tok/s)", count, (decodeStart - prefillStart) * 1e3, steps, (decodeEnd - decodeStart) * 1e3, steps / (decodeEnd - decodeStart));
}
return 0;
}
CC = clang
CFLAGS = -fobjc-arc -O2 -Wall
METAL = -framework Foundation -framework Metal
all: train infer
train: training_objc.m training.metal
$(CC) $(CFLAGS) $(METAL) training_objc.m -o train
infer: inference_objc.m inference.metal
$(CC) $(CFLAGS) $(METAL) inference_objc.m -o infer
clean:
rm -f train infer
.PHONY: all clean