init

Function parameters

Parameters

buffer:[:0]const u8

Type definitions in this namespace

Types

Token
Tokenizer

Source

Implementation

pub fn init(buffer: [:0]const u8) Tokenizer {
    // Skip the UTF-8 BOM if present.
    return .{
        .buffer = buffer,
        .index = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0,
    };
}