# Foundation Before Tokenizers *The model doesn't read text. It reads numbers. Before we get to tokenizers, we need to understand something more fundamental - what IS text inside a machine?* You type "hello". The model does math. Math needs numbers. "hello" is not a number. We need something that chops the text and assign each piece a number. That something is called a **tokenizer**. But to understand why tokenizers work the way they do, we need to start one level deeper - what IS text inside a machine? What is data for machines! We'll trace the path: bits → storing letters (ASCII) → Unicode → bytes → UTF-8 → tokenization. The actual problem of chopping text into pieces a model can use. But lets start from bottom. It is needed to build a clear understanding. #dt-wrap{font-family:'IBM Plex Mono',monospace;font-size:10px;max-width:520px;margin:1em auto;} #dt-main{display:flex;gap:0;align-items:stretch;} .dt-side{width:42%;display:flex;flex-direction:column;} .dt-side-label{color:#64748b;font-size:9px;text-transform:uppercase;letter-spacing:0.1em;margin-bottom:4px;font-weight:600;} #dt-left .dt-side-label{text-align:left;} #dt-right .dt-side-label{text-align:right;} #dt-right .dt-row{text-align:right;} #dt-right .dt-lbl{text-align:right;} #dt-right .dt-val{text-align:right;direction:ltr;} .dt-row{margin:2px 0;opacity:0;transition:opacity 0.3s;overflow:hidden;} .dt-row.dt-show{opacity:1;} .dt-lbl{color:#64748b;font-size:9px;display:block;margin-bottom:1px;} .dt-val{color:#94a3b8;font-size:10px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;} .dt-g{color:#4ade80;}.dt-p{color:#f472b6;}.dt-y{color:#e5c07b;} #dt-mid{width:16%;display:flex;flex-direction:column;align-items:center;justify-content:flex-end;position:relative;} #dt-wire-box{width:100%;position:relative;margin-bottom:8px;} #dt-wire{height:18px;width:100%;background:#0a0f1a;border:1px solid #334155;border-radius:9px;position:relative;overflow:hidden;display:flex;align-items:center;} .dt-fly{position:absolute;font-size:9px;color:#4ade80;font-weight:600;font-family:'IBM Plex Mono',monospace;top:3px;animation:dtfly 2s linear forwards;opacity:0;white-space:nowrap;text-shadow:0 0 3px rgba(74,222,128,0.4);} @keyframes dtfly{0%{left:-10px;opacity:0;}5%{opacity:1;}95%{opacity:1;}100%{left:calc(100% + 10px);opacity:0;}} #dt-arrow{color:#4ade80;font-size:11px;margin-bottom:2px;} #dt-input-wrap{text-align:center;margin-top:8px;} #dt-input{background:#1e293b;border:1px solid #334155;border-radius:4px;color:#f1f5f9;font-family:'IBM Plex Mono',monospace;font-size:12px;padding:3px 6px;width:80px;text-align:center;} #dt-input:focus{border-color:#4ade80;outline:none;} #dt-hint{color:#64748b;font-size:9px;margin-top:3px;} <div id="dt-wrap"> <div id="dt-main"> <div class="dt-side" id="dt-left"> <div class="dt-side-label">Sender</div> <div class="dt-row" id="dt-l1"><span class="dt-lbl">Text</span><span class="dt-val" id="dt-lv1"></span></div> <div class="dt-row" id="dt-l2"><span class="dt-lbl">Unicode</span><span class="dt-val" id="dt-lv2"></span></div> <div class="dt-row" id="dt-l3"><span class="dt-lbl">UTF-8</span><span class="dt-val" id="dt-lv3"></span></div> <div class="dt-row" id="dt-l4"><span class="dt-lbl">Binary</span><span class="dt-val" id="dt-lv4"></span></div> </div> <div id="dt-mid"> <div id="dt-arrow">→</div> <div id="dt-wire-box"><div id="dt-wire"></div></div> </div> <div class="dt-side" id="dt-right"> <div class="dt-side-label">Receiver</div> <div class="dt-row" id="dt-r1"><span class="dt-lbl">Text</span><span class="dt-val" id="dt-rv1"></span></div> <div class="dt-row" id="dt-r2"><span class="dt-lbl">Unicode</span><span class="dt-val" id="dt-rv2"></span></div> <div class="dt-row" id="dt-r3"><span class="dt-lbl">UTF-8</span><span class="dt-val" id="dt-rv3"></span></div> <div class="dt-row" id="dt-r4"><span class="dt-lbl">Binary</span><span class="dt-val" id="dt-rv4"></span></div> </div> </div> <div id="dt-input-wrap"> <input id="dt-input" type="text" value="" placeholder="Type..." maxlength="4"> <div id="dt-hint">Auto-playing. Click to type your own.</div> </div> </div> (function(){ var el=function(id){return document.getElementById(id);}; var msgs=['Hi','अ','A','₹','🔥','Ok','नम']; var idx=0,timer=null,paused=false; function reset(){ ['l1','l2','l3','l4','r1','r2','r3','r4'].forEach(function(id){el('dt-'+id).classList.remove('dt-show');}); ['lv1','lv2','lv3','lv4','rv1','rv2','rv3','rv4'].forEach(function(id){el('dt-'+id).innerHTML='';}); el('dt-wire').innerHTML=''; } function getBytes(s){var enc=new TextEncoder();return Array.from(enc.encode(s));} function getCPs(s){var r=[];for(var i=0;i<s.length;){var cp=s.codePointAt(i);r.push(cp);i+=cp>0xFFFF?2:1;}return r;} function animate(text){ reset(); var bytes=getBytes(text); var cps=getCPs(text); var bins=bytes.map(function(b){return b.toString(2).padStart(8,'0');}); var cpStr=cps.map(function(c){return '<span class="dt-g">'+c+'</span>';}).join(' '); var byteStr=bytes.map(function(b){return '<span class="dt-g">'+b+'</span>';}).join(' '); var binStr=bins.map(function(b){return '<span class="dt-p">'+b+'</span>';}).join(' '); setTimeout(function(){ el('dt-lv1').innerHTML='<span class="dt-y">'+text+'</span>'; el('dt-l1').classList.add('dt-show'); },150); setTimeout(function(){ el('dt-lv2').innerHTML=cpStr; el('dt-l2').classList.add('dt-show'); },500); setTimeout(function(){ el('dt-lv3').innerHTML=byteStr; el('dt-l3').classList.add('dt-show'); },850); setTimeout(function(){ el('dt-lv4').innerHTML=binStr; el('dt-l4').classList.add('dt-show'); },1200); setTimeout(function(){ var wire=el('dt-wire'); bins.forEach(function(b,i){ var span=document.createElement('span'); span.className='dt-fly'; span.textContent=b; span.style.animationDelay=(i*0.35)+'s'; wire.appendChild(span); }); },1600); setTimeout(function(){ el('dt-rv4').innerHTML=binStr; el('dt-r4').classList.add('dt-show'); },2400); setTimeout(function(){ el('dt-rv3').innerHTML=byteStr; el('dt-r3').classList.add('dt-show'); },2700); setTimeout(function(){ el('dt-rv2').innerHTML=cpStr; el('dt-r2').classList.add('dt-show'); },3000); setTimeout(function(){ el('dt-rv1').innerHTML='<span class="dt-y">'+text+'</span> <span class="dt-g">✓</span>'; el('dt-r1').classList.add('dt-show'); },3300); } function next(){ if(paused)return; animate(msgs[idx%msgs.length]); idx++; } function startLoop(){ next(); timer=setInterval(next,4500); } el('dt-input').addEventListener('focus',function(){ paused=true; if(timer){clearInterval(timer);timer=null;} el('dt-hint').textContent='Press Enter to send'; }); el('dt-input').addEventListener('blur',function(){ if(!el('dt-input').value){ paused=false; el('dt-hint').textContent='Auto-playing. Click to type your own.'; startLoop(); } }); el('dt-input').addEventListener('keydown',function(e){ if(e.key==='Enter'&&this.value){animate(this.value);} }); startLoop(); })(); </div> --- ## The bit A computer doesn't have a brain. It has billions of tiny **switches** called [transistors](https://en.wikipedia.org/wiki/Transistor). Each one does exactly one thing: let electricity through, or block it. Only two states: 0 or 1. <span class="key-line">That's a [bit](https://en.wikipedia.org/wiki/Bit). The most basic unit of information.</span> <div style="font-family:'IBM Plex Mono';font-size:13px;color:#94a3b8;text-align:center;margin:1em 0 0.8em;"> <span id="bs-switch" style="cursor:pointer;display:inline-block;width:48px;height:48px;line-height:48px;text-align:center;background:#1e293b;color:#64748b;border:2px solid #334155;border-radius:8px;font-weight:700;font-size:24px;transition:all 0.15s;user-select:none;">0</span> <span id="bs-label" style="margin-left:1em;color:#64748b;font-size:14px;">off - electricity blocked</span> </div> <div style="text-align:center;font-family:'IBM Plex Mono';font-size:11px;color:#64748b;margin:0 0 0.8em;">Click the switch</div> (function(){ var el=function(id){return document.getElementById(id);}; var on=false; el('bs-switch').addEventListener('click',function(){ on=!on; this.textContent=on?'1':'0'; this.style.background=on?'#4ade80':'#1e293b'; this.style.color=on?'#0a0f1a':'#64748b'; this.style.borderColor=on?'#4ade80':'#334155'; el('bs-label').textContent=on?'on - electricity flowing':'off - electricity blocked'; el('bs-label').style.color=on?'#4ade80':'#64748b'; }); })(); </div> You might wonder - why not have ten levels of electricity? Like our decimal system, 0 to 9? Because electricity is messy. Telling apart 0.45 volts from 0.55 volts is unreliable. Heat, noise, tiny manufacturing differences can cause miscalculations, not reliable. But two states? On or off? Wide gap. Hard to get wrong. Building a machine that only needs to recognize two states is significantly cheaper and easier than building one that recognizes many. Hence 0 and 1. One bit stores a 0 or a 1 - just two values. To represent the letters(a-z, A-Z), digits(0-9), and symbols we use every day, we need far more than two. <span class="key-line">We needed more bits.</span> --- ## Storing letters Ok so the computer only understands 0s and 1s. But we need to store text - letters, digits, symbols. How? The most natural idea: assign each character a number. 'A' gets 65. 'B' gets 66. 'z' gets 122. Make a table, get everyone to agree on it, and move on. That's exactly what happened. In the 1960s, someone sat down and made a table - 128 entries covering English letters, digits, and common symbols. They called it <span class="key-line">[**ASCII**](https://en.wikipedia.org/wiki/ASCII)</span>. Every computer manufacturer agreed on it. Problem solved - we have a table to convert text to numbers. This worked for English. But ASCII only has **128 slots** - no room for 'अ', 'ñ', '中', or '🔥'. So every country made their own table - India had one, Japan had one, Russia had one. The problem? **Same number meant different characters in different tables.** A file written in India showed garbage when opened in Japan. <span class="key-line">There was no single agreed-upon table for the world's text.</span> The fix is obvious in hindsight: <span class="key-line">make ONE table for everyone</span>. One table with every character from every script in every language. | Character | Code Point | |---|---| | A | 65 | | अ | 2,309 | | 中 | 20,013 | | 🔥 | 128,293 | This table is called <span class="key-line">[**Unicode**](https://en.wikipedia.org/wiki/Unicode)</span>. 150,000+ entries and [still growing](https://home.unicode.org/about-unicode/). Each character's number is its <span class="key-line">[**code point**](https://en.wikipedia.org/wiki/Code_point)</span>. <div style="font-family:'IBM Plex Mono';font-size:12px;max-width:420px;margin:0.5em auto;"> <table id="ue-table" style="width:100%;border-collapse:collapse;margin:0.5em 0;"> <tr style="border-bottom:1px solid #334155;"> <td style="padding:6px 8px;color:#64748b;white-space:nowrap;">Character</td> <td id="ue-char" style="padding:6px 8px;font-size:28px;"></td> </tr> <tr style="border-bottom:1px solid #334155;"> <td style="padding:6px 8px;color:#64748b;white-space:nowrap;">Code Point</td> <td id="ue-cp" style="padding:6px 8px;color:#4ade80;font-size:14px;"></td> </tr> <tr style="border-bottom:1px solid #334155;"> <td style="padding:6px 8px;color:#64748b;white-space:nowrap;">Hex</td> <td id="ue-dec" style="padding:6px 8px;color:#94a3b8;"></td> </tr> <tr style="border-bottom:1px solid #334155;"> <td style="padding:6px 8px;color:#64748b;white-space:nowrap;">In ASCII?</td> <td id="ue-ascii" style="padding:6px 8px;"></td> </tr> <tr style="background:rgba(74,222,128,0.06);"> <td style="padding:6px 8px;color:#4ade80;white-space:nowrap;font-weight:700;">Try It</td> <td style="padding:6px 8px;"><input id="ue-input" type="text" value="A" maxlength="2" style="background:#1e293b;border:1px solid #4ade80;border-radius:4px;color:#f1f5f9;font-family:'IBM Plex Mono';font-size:16px;padding:4px 8px;width:50px;text-align:center;"> <span style="color:#64748b;font-size:11px;">type: अ, 中, 🔥, ñ, €, ₹</span></td> </tr> </table> </div> (function(){ var el=function(id){return document.getElementById(id);}; function update(){ var s=el('ue-input').value; if(!s){el('ue-char').textContent='';el('ue-cp').textContent='';el('ue-dec').textContent='';el('ue-ascii').textContent='';return;} var cp=s.codePointAt(0); var ch=String.fromCodePoint(cp); el('ue-char').textContent=ch; el('ue-cp').textContent=cp.toString(10); el('ue-dec').innerHTML='<span style="color:#e5c07b;">U+'+cp.toString(16).toUpperCase().padStart(4,'0')+'</span>'; if(cp<128){ el('ue-ascii').innerHTML='<span style="color:#4ade80;">Yes</span> <span style="color:#64748b;">- one of the original 128</span>'; } else { el('ue-ascii').innerHTML='<span style="color:#f472b6;">No</span> <span style="color:#64748b;">- needs the full Unicode table</span>'; } } el('ue-input').addEventListener('input',update); update(); })(); </div> --- ## The byte We have a number for every character. 'A' is 65, '!' is 33, '🔥' is 128,293. But the machine only stores 0s and 1s - bits. So we need to convert these numbers into bits. 65 in bits is 1000001. That's 7 bits. But if I send these bits to another computer, how does it know where one number ends and the next begins? If my computer groups bits in 7s and yours groups in 8s, you'll read completely different numbers from the same bits. <span class="key-line">Everyone needs to agree on one fixed group size.</span> And that's the beautiful thing about bytes - the entire world agreed on one number. <span class="key-line">8 bits = 1 [**byte**](https://en.wikipedia.org/wiki/Byte).</span> Every computer, every operating system, every file - same group size. That's what makes data portable. But why 8? <div style="font-family:'IBM Plex Mono';font-size:13px;color:#94a3b8;text-align:center;margin:1em 0 0.5em;"> <span style="color:#64748b;">bits:</span> <span id="be-bits" style="display:inline-flex;gap:4px;margin:0 0.5em;"></span> <span style="color:#64748b;">=</span> <input id="be-dec-input" type="number" min="0" max="255" value="65" style="background:#1e293b;border:1px solid #334155;border-radius:4px;color:#4ade80;font-family:'IBM Plex Mono';font-size:13px;padding:2px 6px;width:52px;text-align:center;"> <span style="color:#64748b;margin-left:0.5em;">range: <span id="be-range" style="color:#e5c07b;">0-255</span></span> </div> <div style="text-align:center;margin:0.3em 0 0.5em;"> <label style="font-family:'IBM Plex Mono';font-size:12px;color:#94a3b8;display:flex;align-items:center;gap:0.5em;justify-content:center;"> number of bits <span id="be-nv" style="color:#e5c07b;min-width:1.5em;">8</span> <input id="be-ns" type="range" min="1" max="8" value="8" step="1" style="width:160px;accent-color:#e5c07b;"> </label> </div> <div style="text-align:center;font-family:'IBM Plex Mono';font-size:11px;color:#64748b;margin:0.3em 0 0.3em;">Click bits to toggle, or type a number</div> (function(){ var el=function(id){return document.getElementById(id);}; var n=8, bits=[0,1,0,0,0,0,0,1]; function getVal(){ var val=0; for(var i=0;i<n;i++) val+=bits[i]*Math.pow(2,n-1-i); return val; } function setVal(v){ v=Math.max(0,Math.min(Math.pow(2,n)-1,v)); for(var i=n-1;i>=0;i--){ bits[i]=v%2; v=Math.floor(v/2); } } function render(){ n=parseInt(el('be-ns').value); el('be-nv').textContent=n; var max=Math.pow(2,n)-1; el('be-dec-input').max=max; var html=''; for(var i=0;i<n;i++){ html+='<span data-i="'+i+'" style="cursor:pointer;display:inline-block;width:22px;height:26px;line-height:26px;text-align:center;background:'+(bits[i]?'#4ade80':'#1e293b')+';color:'+(bits[i]?'#0a0f1a':'#64748b')+';border:1px solid '+(bits[i]?'#4ade80':'#334155')+';border-radius:4px;font-weight:600;font-size:14px;">'+bits[i]+'</span>'; } el('be-bits').innerHTML=html; var val=getVal(); el('be-dec-input').value=val; el('be-range').textContent='0-'+max+' ('+Math.pow(2,n)+' values)'; el('be-bits').querySelectorAll('span').forEach(function(s){ s.addEventListener('click',function(){ var idx=parseInt(this.getAttribute('data-i')); bits[idx]=1-bits[idx]; render(); }); }); } el('be-ns').addEventListener('input',function(){ for(var i=0;i<8;i++) bits[i]=0; render(); }); el('be-dec-input').addEventListener('input',function(){ var v=parseInt(this.value)||0; if(v>Math.pow(2,n)-1){ var needed=Math.ceil(Math.log2(v+1)); if(needed>8) needed=8; if(needed<1) needed=1; n=needed; el('be-ns').value=n; } setVal(v); render(); }); render(); })(); </div> 1 bit gives you 2 values. 4 bits give you 16 - not enough for the alphabet. 8 bits give you 256 - enough for every letter, digit, and symbol in ASCII, with room to spare. Go beyond 8 and you waste memory storing zeros. <span class="key-line">8 was the sweet spot.</span> One byte holds one number. One number maps to one character: <div style="font-family:'IBM Plex Mono';font-size:12px;max-width:420px;margin:0.5em auto;"> <table style="width:100%;border-collapse:collapse;margin:0.5em 0;"> <tr style="border-bottom:1px solid #334155;"> <td style="padding:6px 8px;color:#64748b;font-weight:600;">Character</td> <td style="padding:6px 8px;color:#64748b;font-weight:600;">ASCII Number</td> <td style="padding:6px 8px;color:#64748b;font-weight:600;">Byte (8 bits)</td> </tr> <tr style="border-bottom:1px solid #334155;"> <td style="padding:6px 8px;color:#f1f5f9;font-size:16px;">A</td> <td style="padding:6px 8px;color:#4ade80;">65</td> <td style="padding:6px 8px;color:#94a3b8;">01000001</td> </tr> <tr style="border-bottom:1px solid #334155;"> <td style="padding:6px 8px;color:#f1f5f9;font-size:16px;">z</td> <td style="padding:6px 8px;color:#4ade80;">122</td> <td style="padding:6px 8px;color:#94a3b8;">01111010</td> </tr> <tr style="border-bottom:1px solid #334155;"> <td style="padding:6px 8px;color:#f1f5f9;font-size:16px;">7</td> <td style="padding:6px 8px;color:#4ade80;">55</td> <td style="padding:6px 8px;color:#94a3b8;">00110111</td> </tr> <tr style="border-bottom:1px solid #334155;"> <td style="padding:6px 8px;color:#f1f5f9;font-size:16px;">!</td> <td style="padding:6px 8px;color:#4ade80;">33</td> <td style="padding:6px 8px;color:#94a3b8;">00100001</td> </tr> <tr style="background:rgba(74,222,128,0.06);"> <td style="padding:6px 8px;color:#4ade80;font-weight:700;">Try It <input id="bx-input" type="text" value="S" maxlength="1" style="background:#1e293b;border:1px solid #4ade80;border-radius:4px;color:#f1f5f9;font-family:'IBM Plex Mono';font-size:16px;padding:4px 8px;width:40px;text-align:center;margin-left:6px;"></td> <td id="bx-num" style="padding:6px 8px;color:#4ade80;"></td> <td id="bx-bits" style="padding:6px 8px;color:#94a3b8;"><span id="bx-byte"></span></td> </tr> </table> </div> (function(){ var el=function(id){return document.getElementById(id);}; function update(){ var s=el('bx-input').value; if(!s){el('bx-num').textContent='';el('bx-byte').textContent='';return;} var code=s.charCodeAt(0); el('bx-num').textContent=code; if(code<256){ el('bx-byte').textContent=code.toString(2).padStart(8,'0'); } else { el('bx-byte').textContent='(needs more than 1 byte)'; } } el('bx-input').addEventListener('input',update); update(); })(); </div> So one byte stores one ASCII character. 'A' is 65, fits in a byte. 'z' is 122, fits in a byte. Every English letter, digit, and symbol - one byte each. Type "Hello" and that's 5 bytes sitting in memory. <span class="key-line">This worked perfectly. Until we tried to store Unicode.</span> --- ## Code points don't fit in bytes Unicode gave 'अ' the number 2,309. But a byte only holds 0 to 255. <span class="key-line">That number doesn't fit in one byte.</span> Split it across multiple bytes: | Character | Unicode Number | Sequence of Bytes | |---|---|---| | A | 65 | [65] | | अ | 2,309 | [224, 164, 133] | | ₹ | 8,377 | [226, 130, 185] | | 🔥 | 128,293 | [240, 159, 148, 165] | Bigger the number, more bytes needed. But there's a problem. If the receiver gets a stream of bytes like [224, 164, 133, 65, 226, 130, 185] - how does it know that the first three bytes are one character (अ), the next one byte is another (A), and the last three are another (₹)? Without a rule, bytes are just a stream of numbers with no boundaries. Someone had to define exactly how to split a Unicode number into bytes, and how to read them back. <span class="key-line">That rule is called an **encoding**.</span> The world tried three. [**UTF-32**](https://en.wikipedia.org/wiki/UTF-32) was the simplest: always use 4 bytes per character, no matter what. 'A' gets 4 bytes, 'अ' gets 4 bytes. Easy to decode - every 4 bytes is one character. But 'A' is just 65. It needs 1 byte. You're wasting 3 bytes on zeros. English text becomes 4x bigger for no reason. [**UTF-16**](https://en.wikipedia.org/wiki/UTF-16) was the compromise. Windows and Java adopted it in the 1990s, betting that 2 bytes (65,536 values) would cover every character. It didn't - emojis alone blew past that limit. [**UTF-8**](https://en.wikipedia.org/wiki/UTF-8) was the clever one. Use only as many bytes as you need. How? It uses bit templates. Each template reserves some bits as markers (so the decoder knows how many bytes to expect) and the rest carry the actual number: | Unicode Range | Bytes | Bit Template | Data Bits | |---|---|---|---| | 0 - 127 | 1 | `0xxxxxxx` | 7 | | 128 - 2,047 | 2 | `110xxxxx 10xxxxxx` | 11 | | 2,048 - 65,535 | 3 | `1110xxxx 10xxxxxx 10xxxxxx` | 16 | | 65,536 - 1,114,111 | 4 | `11110xxx 10xxxxxx 10xxxxxx 10xxxxxx` | 21 | The `x` slots are where the actual number goes. The `1`s and `0`s at the start are fixed markers. Let's trace 'अ' (Unicode 2,309). It falls in the 2,048 - 65,535 range, so it needs 3 bytes. The 3-byte template has 16 data slots (4 + 6 + 6): **Step 1** - convert 2,309 to binary and pad to 16 bits: ``` 2,309 → 0000 100100 000101 ``` **Step 2** - split into groups of 4, 6, 6 to match the template slots: ``` 0000 | 100100 | 000101 ↑ ↑ ↑ 4 bits 6 bits 6 bits ``` **Step 3** - place into the template (markers shown in bold): ``` Template: 1110 xxxx 10 xxxxxx 10 xxxxxx Data: 0000 100100 000101 Result: 1110 0000 10 100100 10 000101 Decimal: 224 164 133 ``` That's how 'अ' becomes [224, 164, 133]. When the decoder reads 224 (`11100000`), the leading `1110` tells it: "I am the start of a 3-byte character - read the next 2 bytes with me." 'A' is 65 - fits in the first row, one byte, no splitting needed. The first 128 values are identical to ASCII, so every old program still works (backward compatible). <span class="key-line">UTF-8 won. [98% of the web](https://w3techs.com/technologies/details/en-utf8) uses it.</span> Try it yourself - type any character and watch the encoding happen step by step: #u8e-wrap{font-family:'IBM Plex Mono',monospace;font-size:12px;max-width:520px;margin:0.5em auto;} #u8e-input-row{text-align:center;margin:0.8em 0 0.5em;} #u8e-char-input{background:#1e293b;border:1px solid #4ade80;border-radius:4px;color:#f1f5f9;font-family:'IBM Plex Mono',monospace;font-size:18px;padding:4px 8px;width:50px;text-align:center;} #u8e-char-input:focus{outline:none;border-color:#4ade80;} .u8e-step{margin:6px 0;padding:6px 8px;border-radius:4px;opacity:0;transform:translateY(4px);transition:opacity 0.3s,transform 0.3s;} .u8e-step.u8e-show{opacity:1;transform:translateY(0);} .u8e-label{color:#64748b;font-size:11px;margin-bottom:2px;} .u8e-val{font-size:12px;} .u8e-green{color:#4ade80;} .u8e-pink{color:#f472b6;} .u8e-yellow{color:#e5c07b;} .u8e-muted{color:#64748b;} .u8e-marker{color:#64748b;font-weight:400;} .u8e-data{color:#4ade80;font-weight:600;} .u8e-template-row{display:flex;gap:6px;flex-wrap:wrap;margin:2px 0;} .u8e-bit{display:inline-block;width:14px;height:18px;line-height:18px;text-align:center;font-size:11px;border-radius:2px;opacity:0;transition:opacity 0.15s;} .u8e-bit.u8e-show{opacity:1;} .u8e-bit.u8e-m{background:#1e293b;color:#64748b;border:1px solid #334155;} .u8e-bit.u8e-d{background:#1a3a2a;color:#4ade80;border:1px solid #4ade80;} #u8e-result{margin-top:8px;padding:8px;background:rgba(74,222,128,0.06);border-radius:4px;text-align:center;opacity:0;transition:opacity 0.3s;} #u8e-result.u8e-show{opacity:1;} <div id="u8e-wrap"> <div id="u8e-input-row"> <span style="color:#64748b;">Type a character:</span> <input id="u8e-char-input" type="text" value="अ" maxlength="2"> <button id="u8e-play" style="background:#334155;color:#94a3b8;border:none;border-radius:4px;padding:4px 10px;font-family:inherit;font-size:11px;cursor:pointer;margin-left:6px;">Replay</button> </div> <div id="u8e-steps"> <div class="u8e-step" id="u8e-s1"><div class="u8e-label">Character & Code Point</div><div class="u8e-val" id="u8e-v1"></div></div> <div class="u8e-step" id="u8e-s2"><div class="u8e-label">Binary (padded)</div><div class="u8e-val" id="u8e-v2"></div></div> <div class="u8e-step" id="u8e-s3"><div class="u8e-label">UTF-8 Template</div><div class="u8e-val" id="u8e-v3"></div></div> <div class="u8e-step" id="u8e-s4"><div class="u8e-label">Fill the template</div><div class="u8e-val" id="u8e-v4"></div></div> </div> <div id="u8e-result"></div> </div> (function(){ var el=function(id){return document.getElementById(id);}; var templates=[ {max:0x7F,bytes:1,markers:['0'],slots:[7],prefix:'0'}, {max:0x7FF,bytes:2,markers:['110','10'],slots:[5,6],prefix:'110xxxxx 10xxxxxx'}, {max:0xFFFF,bytes:3,markers:['1110','10','10'],slots:[4,6,6],prefix:'1110xxxx 10xxxxxx 10xxxxxx'}, {max:0x10FFFF,bytes:4,markers:['11110','10','10','10'],slots:[3,6,6,6],prefix:'11110xxx 10xxxxxx 10xxxxxx 10xxxxxx'} ]; function findTemplate(cp){ for(var i=0;i<templates.length;i++){if(cp<=templates[i].max)return templates[i];} return templates[3]; } function reset(){ for(var i=1;i<=4;i++){el('u8e-s'+i).classList.remove('u8e-show');el('u8e-v'+i).innerHTML='';} el('u8e-result').classList.remove('u8e-show');el('u8e-result').innerHTML=''; el('u8e-v4').innerHTML=''; } function animate(){ reset(); var s=el('u8e-char-input').value; if(!s)return; var cp=s.codePointAt(0); var ch=String.fromCodePoint(cp); var tmpl=findTemplate(cp); var totalSlots=0; tmpl.slots.forEach(function(s){totalSlots+=s;}); var bin=cp.toString(2); while(bin.length<totalSlots)bin='0'+bin; var enc=new TextEncoder(); var bytes=Array.from(enc.encode(ch)); setTimeout(function(){ el('u8e-v1').innerHTML='<span style="font-size:20px;">'+ch+'</span> <span class="u8e-muted">=</span> <span class="u8e-green">'+cp+'</span> <span class="u8e-muted">(needs '+tmpl.bytes+' byte'+(tmpl.bytes>1?'s':'')+')</span>'; el('u8e-s1').classList.add('u8e-show'); },200); setTimeout(function(){ var groups=[]; var pos=0; tmpl.slots.forEach(function(s){ groups.push(bin.substring(pos,pos+s)); pos+=s; }); el('u8e-v2').innerHTML='<span class="u8e-green">'+groups.join(' ')+'</span> <span class="u8e-muted">('+bin.length+' bits)</span>'; el('u8e-s2').classList.add('u8e-show'); },800); setTimeout(function(){ el('u8e-v3').innerHTML='<span class="u8e-muted">'+tmpl.prefix+'</span>'; el('u8e-s3').classList.add('u8e-show'); },1400); setTimeout(function(){ var html=''; var bitPos=0; var byteIdx=0; tmpl.markers.forEach(function(marker,mi){ if(mi>0)html+='<span style="display:inline-block;width:6px;"></span>'; for(var j=0;j<marker.length;j++){ html+='<span class="u8e-bit u8e-m u8e-show">'+marker[j]+'</span>'; } var slotCount=tmpl.slots[mi]; for(var k=0;k<slotCount;k++){ var b=bin[bitPos++]; html+='<span class="u8e-bit u8e-d" style="transition-delay:'+(bitPos*0.06)+'s;">'+b+'</span>'; } }); el('u8e-v4').innerHTML=html; setTimeout(function(){ el('u8e-v4').querySelectorAll('.u8e-d').forEach(function(b){b.classList.add('u8e-show');}); },50); },2000); setTimeout(function(){ el('u8e-result').innerHTML='<span class="u8e-green" style="font-size:14px;">['+bytes.join(', ')+']</span> <span class="u8e-muted">= '+ch+'</span>'; el('u8e-result').classList.add('u8e-show'); },3000); } el('u8e-char-input').addEventListener('input',function(){animate();}); el('u8e-play').addEventListener('click',function(){animate();}); animate(); })(); </div> But how does UTF-8 know 'अ' needs three bytes? And how does the decoder know where one character ends and the next begins? --- ## How UTF-8 actually works Unicode is a table — it assigns 'अ' the number 2,309. UTF-8 is the encoding — the rule for storing that number as bytes. Two separate things. **Encoding 'अ':** UTF-8 has bit templates. Leading bits are fixed markers. The `x` slots carry the actual number: ``` 1 byte: 0xxxxxxx → 7 slots → fits 0–127 2 bytes: 110xxxxx 10xxxxxx → 11 slots → fits 128–2,047 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx → 16 slots → fits 2,048–65,535 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx → 21 slots → fits 65,536–1,114,111 ``` The markers eat some bits. The leftover `x` slots determine the max value each template holds — and that max IS the range boundary. Encoder checks: is 2,309 ≤ 127? No. ≤ 2,047? No. ≤ 65,535? Yes → 3 bytes. Then fills the template: ``` 2,309 in binary: 0000 1001 0000 0101 Template: 1110xxxx 10xxxxxx 10xxxxxx Filled: 1110 0000 10 100100 10 000101 Hex: E0 A4 85 ``` **Decoding — how leading bits mark boundaries:** Byte stream arrives: `E0 A4 85 41` ``` E0 → 11100000 starts with 1110 → "I start a 3-byte char, read 2 more" A4 → 10100100 starts with 10 → continuation ✓ 85 → 10000101 starts with 10 → continuation ✓ → those three = 'अ' 41 → 01000001 starts with 0 → "single byte, I'm alone" → that one = 'A' ``` A byte starting with `10` is never a start byte. A byte starting with `1110` is never a continuation. No ambiguity — you can drop into the middle of a stream and find the next character boundary by scanning forward until you hit a byte that doesn't start with `10`. <div style="text-align:center;margin:1em 0 0.3em;"> <label style="font-family:'IBM Plex Mono';font-size:12px;color:#94a3b8;display:flex;align-items:center;gap:0.5em;justify-content:center;"> Type anything: <input id="uc-input" type="text" value="Hey आलोक 🔥" style="background:#1e293b;border:1px solid #334155;border-radius:4px;color:#f1f5f9;font-family:'IBM Plex Mono';font-size:13px;padding:4px 8px;width:200px;"> </label> </div> <div style="font-family:'IBM Plex Mono';font-size:12px;margin:0.5em auto;max-width:520px;"> <div style="display:flex;gap:0.5em;margin:0.3em 0;align-items:baseline;"> <span style="color:#4ade80;min-width:52px;">UTF-8</span> <span id="uc-utf8" style="color:#94a3b8;word-break:break-all;"></span> <span id="uc-utf8-n" style="color:#e5c07b;white-space:nowrap;"></span> </div> <div style="display:flex;gap:0.5em;margin:0.3em 0;align-items:baseline;"> <span style="color:#e5c07b;min-width:52px;">UTF-16</span> <span id="uc-utf16" style="color:#94a3b8;word-break:break-all;"></span> <span id="uc-utf16-n" style="color:#e5c07b;white-space:nowrap;"></span> </div> <div style="display:flex;gap:0.5em;margin:0.3em 0;align-items:baseline;"> <span style="color:#f472b6;min-width:52px;">UTF-32</span> <span id="uc-utf32" style="color:#94a3b8;word-break:break-all;"></span> <span id="uc-utf32-n" style="color:#e5c07b;white-space:nowrap;"></span> </div> <div style="display:flex;gap:0.5em;margin:0.5em 0;align-items:baseline;"> <span style="color:#64748b;min-width:52px;">waste</span> <span id="uc-waste" style="color:#e5c07b;"></span> </div> </div> (function(){ var el=function(id){return document.getElementById(id);}; function update(){ var s=el('uc-input').value; var enc=new TextEncoder(); var u8=enc.encode(s); var hex8=Array.from(u8).map(function(b){return b.toString(16).toUpperCase().padStart(2,'0');}).join(' '); el('uc-utf8').textContent=hex8; el('uc-utf8-n').textContent='('+u8.length+' bytes)'; var u16len=0; var hex16parts=[]; var u32len=0; var hex32parts=[]; for(var i=0;i<s.length;){ var cp=s.codePointAt(i); if(cp>0xFFFF){ hex16parts.push(cp.toString(16).toUpperCase().padStart(8,'0')); u16len+=4; } else { hex16parts.push(cp.toString(16).toUpperCase().padStart(4,'0')); u16len+=2; } hex32parts.push(cp.toString(16).toUpperCase().padStart(8,'0')); u32len+=4; i+=cp>0xFFFF?2:1; } el('uc-utf16').textContent=hex16parts.join(' '); el('uc-utf16-n').textContent='('+u16len+' bytes)'; el('uc-utf32').textContent=hex32parts.join(' '); el('uc-utf32-n').textContent='('+u32len+' bytes)'; if(u8.length>0){ var pct=Math.round((1-u8.length/u32len)*100); el('uc-waste').textContent='UTF-32 uses '+(u32len-u8.length)+' extra bytes vs UTF-8 ('+pct+'% overhead)'; } else { el('uc-waste').textContent=''; } } el('uc-input').addEventListener('input',update); update(); })(); </div> --- ## What we know now We started with a simple question: what is text inside a machine? The answer built up layer by layer. Each step solved a problem and created the next: - **Bits** - a machine only has switches. 0 or 1. But you can't store text with just two values. - **ASCII** - assign each character a number. 'A' = 65. Worked for English. But the rest of the world had no room. - **Unicode** - one table for every character in every language. 150,000+ entries. But these numbers are huge and need to be stored in memory. - **Bytes** - group 8 bits together. 256 values. The world agreed on this as the unit of storage. But Unicode numbers like 2,309 don't fit in one byte. - **UTF-8** - split big numbers across multiple bytes, using markers so the receiver knows which bytes belong together. No waste, backward compatible with ASCII. 98% of the web uses it. <span class="key-line">Any text, in any language, is just a sequence of bytes in memory.</span> "Hello" is 5 bytes. "आलोक" is 12 bytes. That's what text is inside a machine. But a language model needs to do math on text. It can't just eat raw bytes (not yet). It needs to break text into pieces and convert each piece into a number the model can work with. That process is called **tokenization** - and it's the subject of the next post. --- *This is part of the [Foundation Model Internals](/blogs) series - a software engineer figuring out how models work, from scratch.*