authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2021-03-19 19:26:30+01:00
committergravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2021-03-28 14:32:34+02:00
logb8c019ef49be54d76acc6721d5d8d493193bcf5d
treef6607c0cc2f6c31736da8bb59b838f13506684dc
parentab9324e604068d4afb4e65a8e587bea95ab1051a

std/base64: cleanups & support url-safe and other non-padded variants

This makes a few changes to the base64 codecs. * The padding character is optional. The common "URL-safe" variant, in particular, is generally not used with padding. This is also the case for password hashes, so having this will avoid code duplication with bcrypt, scrypt and other functions. * The URL-safe variant is added. Instead of having individual constants for each parameter of each variant, we are now grouping these in a struct. So, `standard_pad_char` just becomes `standard.pad_char`. * Types are not `snake_case`'d any more. So, `standard_encoder` becomes `standard.Encoder`, as it is a type. * Creating a decoder with ignored characters required the alphabet and padding. Now, `standard.decoderWithIgnore(<ignored chars>)` returns a decoder with the standard parameters and the set of ignored chars. * Whatever applies to `standard.*` obviously also works with `url_safe.*` * the `calcSize()` interface was inconsistent, taking a length in the encoder, and a slice in the encoder. Rename the variant that takes a slice to `calcSizeForSlice()`. * In the decoder with ignored characters, add `calcSizeUpperBound()`, which is more useful than the one that takes a slice in order to size a fixed buffer before we have the data. * Return `error.InvalidCharacter` when the input actually contains characters that are neither padding nor part of the alphabet. If we hit a padding issue (which includes extra bits at the end), consistently return `error.InvalidPadding`. * Don't keep the `char_in_alphabet` array permanently in a decoder; it is only required for sanity checks during initialization. * Tests are unchanged, but now cover both the standard (padded) and the url-safe (non-padded) variants. * Add an error set, rename `OutputTooSmallError` to `NoSpaceLeft` to match the `hex2bin` equivalent.

5 files changed, 337 insertions(+), 275 deletions(-)

doc/langref.html.in+2-2
...@@ -9952,8 +9952,8 @@ export fn decode_base_64(...@@ -9952,8 +9952,8 @@ export fn decode_base_64(
9952) usize {9952) usize {
9953 const src = source_ptr[0..source_len];9953 const src = source_ptr[0..source_len];
9954 const dest = dest_ptr[0..dest_len];9954 const dest = dest_ptr[0..dest_len];
9955 const base64_decoder = base64.standard_decoder_unsafe;9955 const base64_decoder = base64.standard.DecoderUnsafe;
9956 const decoded_size = base64_decoder.calcSize(src);9956 const decoded_size = base64_decoder.calcSizeForSlice(src) catch unreachable;
9957 base64_decoder.decode(dest[0..decoded_size], src);9957 base64_decoder.decode(dest[0..decoded_size], src);
9958 return decoded_size;9958 return decoded_size;
9959}9959}
lib/std/base64.zig+327-265
...@@ -8,308 +8,339 @@ const assert = std.debug.assert;...@@ -8,308 +8,339 @@ const assert = std.debug.assert;
8const testing = std.testing;8const testing = std.testing;
9const mem = std.mem;9const mem = std.mem;
1010
11pub const standard_alphabet_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";11pub const Error = error{
12pub const standard_pad_char = '=';12 InvalidCharacter,
13pub const standard_encoder = Base64Encoder.init(standard_alphabet_chars, standard_pad_char);13 InvalidPadding,
14 NoSpaceLeft,
15};
16
17/// Base64 codecs
18pub const Codecs = struct {
19 alphabet_chars: [64]u8,
20 pad_char: ?u8,
21 decoderWithIgnore: fn (ignore: []const u8) Base64DecoderWithIgnore,
22 Encoder: Base64Encoder,
23 Decoder: Base64Decoder,
24 DecoderUnsafe: Base64DecoderUnsafe,
25};
26
27pub const standard_alphabet_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".*;
28fn standardBase64DecoderWithIgnore(ignore: []const u8) Base64DecoderWithIgnore {
29 return Base64DecoderWithIgnore.init(standard_alphabet_chars, '=', ignore);
30}
31
32/// Standard Base64 codecs, with padding
33pub const standard = Codecs{
34 .alphabet_chars = standard_alphabet_chars,
35 .pad_char = '=',
36 .decoderWithIgnore = standardBase64DecoderWithIgnore,
37 .Encoder = Base64Encoder.init(standard_alphabet_chars, '='),
38 .Decoder = Base64Decoder.init(standard_alphabet_chars, '='),
39 .DecoderUnsafe = Base64DecoderUnsafe.init(standard_alphabet_chars, '='),
40};
41
42pub const url_safe_alphabet_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".*;
43fn urlSafeBase64DecoderWithIgnore(ignore: []const u8) Base64DecoderWithIgnore {
44 return Base64DecoderWithIgnore.init(url_safe_alphabet_chars, null, ignore);
45}
46
47/// URL-safe Base64 codecs, without padding
48pub const url_safe = Codecs{
49 .alphabet_chars = url_safe_alphabet_chars,
50 .pad_char = null,
51 .decoderWithIgnore = urlSafeBase64DecoderWithIgnore,
52 .Encoder = Base64Encoder.init(url_safe_alphabet_chars, null),
53 .Decoder = Base64Decoder.init(url_safe_alphabet_chars, null),
54 .DecoderUnsafe = Base64DecoderUnsafe.init(url_safe_alphabet_chars, null),
55};
56
57// Backwards compatibility
58
59/// Deprecated - Use `standard.pad_char`
60pub const standard_pad_char = standard.pad_char;
61/// Deprecated - Use `standard.Encoder`
62pub const standard_encoder = standard.Encoder;
63/// Deprecated - Use `standard.Decoder`
64pub const standard_decoder = standard.Decoder;
65/// Deprecated - Use `standard.DecoderUnsafe`
66pub const standard_decoder_unsafe = standard.DecoderUnsafe;
1467
15pub const Base64Encoder = struct {68pub const Base64Encoder = struct {
16 alphabet_chars: []const u8,69 alphabet_chars: [64]u8,
17 pad_char: u8,70 pad_char: ?u8,
1871
19 /// a bunch of assertions, then simply pass the data right through.72 /// A bunch of assertions, then simply pass the data right through.
20 pub fn init(alphabet_chars: []const u8, pad_char: u8) Base64Encoder {73 pub fn init(alphabet_chars: [64]u8, pad_char: ?u8) Base64Encoder {
21 assert(alphabet_chars.len == 64);74 assert(alphabet_chars.len == 64);
22 var char_in_alphabet = [_]bool{false} ** 256;75 var char_in_alphabet = [_]bool{false} ** 256;
23 for (alphabet_chars) |c| {76 for (alphabet_chars) |c| {
24 assert(!char_in_alphabet[c]);77 assert(!char_in_alphabet[c]);
25 assert(c != pad_char);78 assert(pad_char == null or c != pad_char.?);
26 char_in_alphabet[c] = true;79 char_in_alphabet[c] = true;
27 }80 }
28
29 return Base64Encoder{81 return Base64Encoder{
30 .alphabet_chars = alphabet_chars,82 .alphabet_chars = alphabet_chars,
31 .pad_char = pad_char,83 .pad_char = pad_char,
32 };84 };
33 }85 }
3486
35 /// ceil(source_len * 4/3)87 /// Compute the encoded length
36 pub fn calcSize(source_len: usize) usize {88 pub fn calcSize(encoder: *const Base64Encoder, source_len: usize) usize {
37 return @divTrunc(source_len + 2, 3) * 4;89 if (encoder.pad_char != null) {
90 return @divTrunc(source_len + 2, 3) * 4;
91 } else {
92 const leftover = source_len % 3;
93 return @divTrunc(source_len, 3) * 4 + @divTrunc(leftover * 4 + 2, 3);
94 }
38 }95 }
3996
40 /// dest.len must be what you get from ::calcSize.97 /// dest.len must at least be what you get from ::calcSize.
41 pub fn encode(encoder: *const Base64Encoder, dest: []u8, source: []const u8) []const u8 {98 pub fn encode(encoder: *const Base64Encoder, dest: []u8, source: []const u8) []const u8 {
42 assert(dest.len >= Base64Encoder.calcSize(source.len));99 const out_len = encoder.calcSize(source.len);
43100 assert(dest.len >= out_len);
44 var i: usize = 0;101
45 var out_index: usize = 0;102 const nibbles = source.len / 3;
46 while (i + 2 < source.len) : (i += 3) {103 const leftover = source.len - 3 * nibbles;
47 dest[out_index] = encoder.alphabet_chars[(source[i] >> 2) & 0x3f];104
48 out_index += 1;105 var acc: u12 = 0;
49106 var acc_len: u4 = 0;
50 dest[out_index] = encoder.alphabet_chars[((source[i] & 0x3) << 4) | ((source[i + 1] & 0xf0) >> 4)];107 var out_idx: usize = 0;
51 out_index += 1;108 for (source) |v| {
52109 acc = (acc << 8) + v;
53 dest[out_index] = encoder.alphabet_chars[((source[i + 1] & 0xf) << 2) | ((source[i + 2] & 0xc0) >> 6)];110 acc_len += 8;
54 out_index += 1;111 while (acc_len >= 6) {
55112 acc_len -= 6;
56 dest[out_index] = encoder.alphabet_chars[source[i + 2] & 0x3f];113 dest[out_idx] = encoder.alphabet_chars[@truncate(u6, (acc >> acc_len))];
57 out_index += 1;114 out_idx += 1;
115 }
58 }116 }
59117 if (acc_len > 0) {
60 if (i < source.len) {118 dest[out_idx] = encoder.alphabet_chars[@truncate(u6, (acc << 6 - acc_len))];
61 dest[out_index] = encoder.alphabet_chars[(source[i] >> 2) & 0x3f];119 out_idx += 1;
62 out_index += 1;120 }
63121 if (encoder.pad_char) |pad_char| {
64 if (i + 1 == source.len) {122 for (dest[out_idx..]) |*pad| {
65 dest[out_index] = encoder.alphabet_chars[(source[i] & 0x3) << 4];123 pad.* = pad_char;
66 out_index += 1;
67
68 dest[out_index] = encoder.pad_char;
69 out_index += 1;
70 } else {
71 dest[out_index] = encoder.alphabet_chars[((source[i] & 0x3) << 4) | ((source[i + 1] & 0xf0) >> 4)];
72 out_index += 1;
73
74 dest[out_index] = encoder.alphabet_chars[(source[i + 1] & 0xf) << 2];
75 out_index += 1;
76 }124 }
77
78 dest[out_index] = encoder.pad_char;
79 out_index += 1;
80 }125 }
81 return dest[0..out_index];126 return dest[0..out_len];
82 }127 }
83};128};
84129
85pub const standard_decoder = Base64Decoder.init(standard_alphabet_chars, standard_pad_char);
86
87pub const Base64Decoder = struct {130pub const Base64Decoder = struct {
131 const invalid_char: u8 = 0xff;
132
88 /// e.g. 'A' => 0.133 /// e.g. 'A' => 0.
89 /// undefined for any value not in the 64 alphabet chars.134 /// `invalid_char` for any value not in the 64 alphabet chars.
90 char_to_index: [256]u8,135 char_to_index: [256]u8,
136 pad_char: ?u8,
91137
92 /// true only for the 64 chars in the alphabet, not the pad char.138 pub fn init(alphabet_chars: [64]u8, pad_char: ?u8) Base64Decoder {
93 char_in_alphabet: [256]bool,
94 pad_char: u8,
95
96 pub fn init(alphabet_chars: []const u8, pad_char: u8) Base64Decoder {
97 assert(alphabet_chars.len == 64);
98
99 var result = Base64Decoder{139 var result = Base64Decoder{
100 .char_to_index = undefined,140 .char_to_index = [_]u8{invalid_char} ** 256,
101 .char_in_alphabet = [_]bool{false} ** 256,
102 .pad_char = pad_char,141 .pad_char = pad_char,
103 };142 };
104143
144 var char_in_alphabet = [_]bool{false} ** 256;
105 for (alphabet_chars) |c, i| {145 for (alphabet_chars) |c, i| {
106 assert(!result.char_in_alphabet[c]);146 assert(!char_in_alphabet[c]);
107 assert(c != pad_char);147 assert(pad_char == null or c != pad_char.?);
108148
109 result.char_to_index[c] = @intCast(u8, i);149 result.char_to_index[c] = @intCast(u8, i);
110 result.char_in_alphabet[c] = true;150 char_in_alphabet[c] = true;
111 }151 }
152 return result;
153 }
112154
155 /// Return the maximum possible decoded size for a given input length - The actual length may be less if the input includes padding.
156 /// `InvalidPadding` is returned if the input length is not valid.
157 pub fn calcSizeUpperBound(decoder: *const Base64Decoder, source_len: usize) Error!usize {
158 var result = source_len / 4 * 3;
159 const leftover = source_len % 4;
160 if (decoder.pad_char != null) {
161 if (leftover % 4 != 0) return error.InvalidPadding;
162 } else {
163 if (leftover % 4 == 1) return error.InvalidPadding;
164 result += leftover * 3 / 4;
165 }
113 return result;166 return result;
114 }167 }
115168
116 /// If the encoded buffer is detected to be invalid, returns error.InvalidPadding.169 /// Return the exact decoded size for a slice.
117 pub fn calcSize(decoder: *const Base64Decoder, source: []const u8) !usize {170 /// `InvalidPadding` is returned if the input length is not valid.
118 if (source.len % 4 != 0) return error.InvalidPadding;171 pub fn calcSizeForSlice(decoder: *const Base64Decoder, source: []const u8) Error!usize {
119 return calcDecodedSizeExactUnsafe(source, decoder.pad_char);172 const source_len = source.len;
173 var result = try decoder.calcSizeUpperBound(source_len);
174 if (decoder.pad_char) |pad_char| {
175 if (source_len >= 1 and source[source_len - 1] == pad_char) result -= 1;
176 if (source_len >= 2 and source[source_len - 2] == pad_char) result -= 1;
177 }
178 return result;
120 }179 }
121180
122 /// dest.len must be what you get from ::calcSize.181 /// dest.len must be what you get from ::calcSize.
123 /// invalid characters result in error.InvalidCharacter.182 /// invalid characters result in error.InvalidCharacter.
124 /// invalid padding results in error.InvalidPadding.183 /// invalid padding results in error.InvalidPadding.
125 pub fn decode(decoder: *const Base64Decoder, dest: []u8, source: []const u8) !void {184 pub fn decode(decoder: *const Base64Decoder, dest: []u8, source: []const u8) Error!void {
126 assert(dest.len == (decoder.calcSize(source) catch unreachable));185 if (decoder.pad_char != null and source.len % 4 != 0) return error.InvalidPadding;
127 assert(source.len % 4 == 0);186 var acc: u12 = 0;
128187 var acc_len: u4 = 0;
129 var src_cursor: usize = 0;188 var dest_idx: usize = 0;
130 var dest_cursor: usize = 0;189 var leftover_idx: ?usize = null;
131190 for (source) |c, src_idx| {
132 while (src_cursor < source.len) : (src_cursor += 4) {191 const d = decoder.char_to_index[c];
133 if (!decoder.char_in_alphabet[source[src_cursor + 0]]) return error.InvalidCharacter;192 if (d == invalid_char) {
134 if (!decoder.char_in_alphabet[source[src_cursor + 1]]) return error.InvalidCharacter;193 if (decoder.pad_char == null or c != decoder.pad_char.?) return error.InvalidCharacter;
135 if (src_cursor < source.len - 4 or source[src_cursor + 3] != decoder.pad_char) {194 leftover_idx = src_idx;
136 // common case195 break;
137 if (!decoder.char_in_alphabet[source[src_cursor + 2]]) return error.InvalidCharacter;196 }
138 if (!decoder.char_in_alphabet[source[src_cursor + 3]]) return error.InvalidCharacter;197 acc = (acc << 6) + d;
139 dest[dest_cursor + 0] = decoder.char_to_index[source[src_cursor + 0]] << 2 | decoder.char_to_index[source[src_cursor + 1]] >> 4;198 acc_len += 6;
140 dest[dest_cursor + 1] = decoder.char_to_index[source[src_cursor + 1]] << 4 | decoder.char_to_index[source[src_cursor + 2]] >> 2;199 if (acc_len >= 8) {
141 dest[dest_cursor + 2] = decoder.char_to_index[source[src_cursor + 2]] << 6 | decoder.char_to_index[source[src_cursor + 3]];200 acc_len -= 8;
142 dest_cursor += 3;201 dest[dest_idx] = @truncate(u8, acc >> acc_len);
143 } else if (source[src_cursor + 2] != decoder.pad_char) {202 dest_idx += 1;
144 // one pad char
145 if (!decoder.char_in_alphabet[source[src_cursor + 2]]) return error.InvalidCharacter;
146 dest[dest_cursor + 0] = decoder.char_to_index[source[src_cursor + 0]] << 2 | decoder.char_to_index[source[src_cursor + 1]] >> 4;
147 dest[dest_cursor + 1] = decoder.char_to_index[source[src_cursor + 1]] << 4 | decoder.char_to_index[source[src_cursor + 2]] >> 2;
148 if (decoder.char_to_index[source[src_cursor + 2]] << 6 != 0) return error.InvalidPadding;
149 dest_cursor += 2;
150 } else {
151 // two pad chars
152 dest[dest_cursor + 0] = decoder.char_to_index[source[src_cursor + 0]] << 2 | decoder.char_to_index[source[src_cursor + 1]] >> 4;
153 if (decoder.char_to_index[source[src_cursor + 1]] << 4 != 0) return error.InvalidPadding;
154 dest_cursor += 1;
155 }203 }
156 }204 }
157205 if (acc_len > 4 or (acc & (@as(u12, 1) << acc_len) - 1) != 0) {
158 assert(src_cursor == source.len);206 return error.InvalidPadding;
159 assert(dest_cursor == dest.len);207 }
208 if (leftover_idx == null) return;
209 var leftover = source[leftover_idx.?..];
210 if (decoder.pad_char) |pad_char| {
211 const padding_len = acc_len / 2;
212 var padding_chars: usize = 0;
213 var i: usize = 0;
214 for (leftover) |c| {
215 if (c != pad_char) {
216 return if (c == Base64Decoder.invalid_char) error.InvalidCharacter else error.InvalidPadding;
217 }
218 padding_chars += 1;
219 }
220 if (padding_chars != padding_len) return error.InvalidPadding;
221 }
160 }222 }
161};223};
162224
163pub const Base64DecoderWithIgnore = struct {225pub const Base64DecoderWithIgnore = struct {
164 decoder: Base64Decoder,226 decoder: Base64Decoder,
165 char_is_ignored: [256]bool,227 char_is_ignored: [256]bool,
166 pub fn init(alphabet_chars: []const u8, pad_char: u8, ignore_chars: []const u8) Base64DecoderWithIgnore {228
229 pub fn init(alphabet_chars: [64]u8, pad_char: ?u8, ignore_chars: []const u8) Base64DecoderWithIgnore {
167 var result = Base64DecoderWithIgnore{230 var result = Base64DecoderWithIgnore{
168 .decoder = Base64Decoder.init(alphabet_chars, pad_char),231 .decoder = Base64Decoder.init(alphabet_chars, pad_char),
169 .char_is_ignored = [_]bool{false} ** 256,232 .char_is_ignored = [_]bool{false} ** 256,
170 };233 };
171
172 for (ignore_chars) |c| {234 for (ignore_chars) |c| {
173 assert(!result.decoder.char_in_alphabet[c]);235 assert(result.decoder.char_to_index[c] == Base64Decoder.invalid_char);
174 assert(!result.char_is_ignored[c]);236 assert(!result.char_is_ignored[c]);
175 assert(result.decoder.pad_char != c);237 assert(result.decoder.pad_char != c);
176 result.char_is_ignored[c] = true;238 result.char_is_ignored[c] = true;
177 }239 }
178
179 return result;240 return result;
180 }241 }
181242
182 /// If no characters end up being ignored or padding, this will be the exact decoded size.243 /// Return the maximum possible decoded size for a given input length - The actual length may be less if the input includes padding
183 pub fn calcSizeUpperBound(encoded_len: usize) usize {244 /// `InvalidPadding` is returned if the input length is not valid.
184 return @divTrunc(encoded_len, 4) * 3;245 pub fn calcSizeUpperBound(decoder_with_ignore: *const Base64DecoderWithIgnore, source_len: usize) Error!usize {
246 var result = source_len / 4 * 3;
247 if (decoder_with_ignore.decoder.pad_char == null) {
248 const leftover = source_len % 4;
249 result += leftover * 3 / 4;
250 }
251 return result;
185 }252 }
186253
187 /// Invalid characters that are not ignored result in error.InvalidCharacter.254 /// Invalid characters that are not ignored result in error.InvalidCharacter.
188 /// Invalid padding results in error.InvalidPadding.255 /// Invalid padding results in error.InvalidPadding.
189 /// Decoding more data than can fit in dest results in error.OutputTooSmall. See also ::calcSizeUpperBound.256 /// Decoding more data than can fit in dest results in error.NoSpaceLeft. See also ::calcSizeUpperBound.
190 /// Returns the number of bytes written to dest.257 /// Returns the number of bytes written to dest.
191 pub fn decode(decoder_with_ignore: *const Base64DecoderWithIgnore, dest: []u8, source: []const u8) !usize {258 pub fn decode(decoder_with_ignore: *const Base64DecoderWithIgnore, dest: []u8, source: []const u8) Error!usize {
192 const decoder = &decoder_with_ignore.decoder;259 const decoder = &decoder_with_ignore.decoder;
193260 var acc: u12 = 0;
194 var src_cursor: usize = 0;261 var acc_len: u4 = 0;
195 var dest_cursor: usize = 0;262 var dest_idx: usize = 0;
196263 var leftover_idx: ?usize = null;
197 while (true) {264 for (source) |c, src_idx| {
198 // get the next 4 chars, if available265 if (decoder_with_ignore.char_is_ignored[c]) continue;
199 var next_4_chars: [4]u8 = undefined;266 const d = decoder.char_to_index[c];
200 var available_chars: usize = 0;267 if (d == Base64Decoder.invalid_char) {
201 var pad_char_count: usize = 0;268 if (decoder.pad_char == null or c != decoder.pad_char.?) return error.InvalidCharacter;
202 while (available_chars < 4 and src_cursor < source.len) {269 leftover_idx = src_idx;
203 var c = source[src_cursor];270 break;
204 src_cursor += 1;
205
206 if (decoder.char_in_alphabet[c]) {
207 // normal char
208 next_4_chars[available_chars] = c;
209 available_chars += 1;
210 } else if (decoder_with_ignore.char_is_ignored[c]) {
211 // we're told to skip this one
212 continue;
213 } else if (c == decoder.pad_char) {
214 // the padding has begun. count the pad chars.
215 pad_char_count += 1;
216 while (src_cursor < source.len) {
217 c = source[src_cursor];
218 src_cursor += 1;
219 if (c == decoder.pad_char) {
220 pad_char_count += 1;
221 if (pad_char_count > 2) return error.InvalidCharacter;
222 } else if (decoder_with_ignore.char_is_ignored[c]) {
223 // we can even ignore chars during the padding
224 continue;
225 } else return error.InvalidCharacter;
226 }
227 break;
228 } else return error.InvalidCharacter;
229 }271 }
230272 acc = (acc << 6) + d;
231 switch (available_chars) {273 acc_len += 6;
232 4 => {274 if (acc_len >= 8) {
233 // common case275 if (dest_idx == dest.len) return error.NoSpaceLeft;
234 if (dest_cursor + 3 > dest.len) return error.OutputTooSmall;276 acc_len -= 8;
235 assert(pad_char_count == 0);277 dest[dest_idx] = @truncate(u8, acc >> acc_len);
236 dest[dest_cursor + 0] = decoder.char_to_index[next_4_chars[0]] << 2 | decoder.char_to_index[next_4_chars[1]] >> 4;278 dest_idx += 1;
237 dest[dest_cursor + 1] = decoder.char_to_index[next_4_chars[1]] << 4 | decoder.char_to_index[next_4_chars[2]] >> 2;
238 dest[dest_cursor + 2] = decoder.char_to_index[next_4_chars[2]] << 6 | decoder.char_to_index[next_4_chars[3]];
239 dest_cursor += 3;
240 continue;
241 },
242 3 => {
243 if (dest_cursor + 2 > dest.len) return error.OutputTooSmall;
244 if (pad_char_count != 1) return error.InvalidPadding;
245 dest[dest_cursor + 0] = decoder.char_to_index[next_4_chars[0]] << 2 | decoder.char_to_index[next_4_chars[1]] >> 4;
246 dest[dest_cursor + 1] = decoder.char_to_index[next_4_chars[1]] << 4 | decoder.char_to_index[next_4_chars[2]] >> 2;
247 if (decoder.char_to_index[next_4_chars[2]] << 6 != 0) return error.InvalidPadding;
248 dest_cursor += 2;
249 break;
250 },
251 2 => {
252 if (dest_cursor + 1 > dest.len) return error.OutputTooSmall;
253 if (pad_char_count != 2) return error.InvalidPadding;
254 dest[dest_cursor + 0] = decoder.char_to_index[next_4_chars[0]] << 2 | decoder.char_to_index[next_4_chars[1]] >> 4;
255 if (decoder.char_to_index[next_4_chars[1]] << 4 != 0) return error.InvalidPadding;
256 dest_cursor += 1;
257 break;
258 },
259 1 => {
260 return error.InvalidPadding;
261 },
262 0 => {
263 if (pad_char_count != 0) return error.InvalidPadding;
264 break;
265 },
266 else => unreachable,
267 }279 }
268 }280 }
269281 if (acc_len > 4 or (acc & (@as(u12, 1) << acc_len) - 1) != 0) {
270 assert(src_cursor == source.len);282 return error.InvalidPadding;
271283 }
272 return dest_cursor;284 const padding_len = acc_len / 2;
285 if (leftover_idx == null) {
286 if (decoder.pad_char != null and padding_len != 0) return error.InvalidPadding;
287 return dest_idx;
288 }
289 var leftover = source[leftover_idx.?..];
290 if (decoder.pad_char) |pad_char| {
291 var padding_chars: usize = 0;
292 var i: usize = 0;
293 for (leftover) |c| {
294 if (decoder_with_ignore.char_is_ignored[c]) continue;
295 if (c != pad_char) {
296 return if (c == Base64Decoder.invalid_char) error.InvalidCharacter else error.InvalidPadding;
297 }
298 padding_chars += 1;
299 }
300 if (padding_chars != padding_len) return error.InvalidPadding;
301 }
302 return dest_idx;
273 }303 }
274};304};
275305
276pub const standard_decoder_unsafe = Base64DecoderUnsafe.init(standard_alphabet_chars, standard_pad_char);
277
278pub const Base64DecoderUnsafe = struct {306pub const Base64DecoderUnsafe = struct {
279 /// e.g. 'A' => 0.307 /// e.g. 'A' => 0.
280 /// undefined for any value not in the 64 alphabet chars.308 /// undefined for any value not in the 64 alphabet chars.
281 char_to_index: [256]u8,309 char_to_index: [256]u8,
282 pad_char: u8,310 pad_char: ?u8,
283311
284 pub fn init(alphabet_chars: []const u8, pad_char: u8) Base64DecoderUnsafe {312 pub fn init(alphabet_chars: [64]u8, pad_char: ?u8) Base64DecoderUnsafe {
285 assert(alphabet_chars.len == 64);
286 var result = Base64DecoderUnsafe{313 var result = Base64DecoderUnsafe{
287 .char_to_index = undefined,314 .char_to_index = undefined,
288 .pad_char = pad_char,315 .pad_char = pad_char,
289 };316 };
290 for (alphabet_chars) |c, i| {317 for (alphabet_chars) |c, i| {
291 assert(c != pad_char);318 assert(pad_char == null or c != pad_char.?);
292 result.char_to_index[c] = @intCast(u8, i);319 result.char_to_index[c] = @intCast(u8, i);
293 }320 }
294 return result;321 return result;
295 }322 }
296323
297 /// The source buffer must be valid.324 /// Return the exact decoded size for a slice.
298 pub fn calcSize(decoder: *const Base64DecoderUnsafe, source: []const u8) usize {325 /// `InvalidPadding` is returned if the input length is not valid.
299 return calcDecodedSizeExactUnsafe(source, decoder.pad_char);326 pub fn calcSizeForSlice(decoder: *const Base64DecoderUnsafe, source: []const u8) Error!usize {
327 const safe_decoder = Base64Decoder{ .char_to_index = undefined, .pad_char = decoder.pad_char };
328 return safe_decoder.calcSizeForSlice(source);
300 }329 }
301330
302 /// dest.len must be what you get from ::calcDecodedSizeExactUnsafe.331 /// dest.len must be what you get from ::calcDecodedSizeExactUnsafe.
303 /// invalid characters or padding will result in undefined values.332 /// invalid characters or padding will result in undefined values.
304 pub fn decode(decoder: *const Base64DecoderUnsafe, dest: []u8, source: []const u8) void {333 pub fn decode(decoder: *const Base64DecoderUnsafe, dest: []u8, source: []const u8) void {
305 assert(dest.len == decoder.calcSize(source));334 assert(dest.len == decoder.calcSizeForSlice(source) catch unreachable);
306335
307 var src_index: usize = 0;336 var src_index: usize = 0;
308 var dest_index: usize = 0;337 var dest_index: usize = 0;
309 var in_buf_len: usize = source.len;338 var in_buf_len: usize = source.len;
310339
311 while (in_buf_len > 0 and source[in_buf_len - 1] == decoder.pad_char) {340 if (decoder.pad_char) |pad_char| {
312 in_buf_len -= 1;341 while (in_buf_len > 0 and source[in_buf_len - 1] == pad_char) {
342 in_buf_len -= 1;
343 }
313 }344 }
314345
315 while (in_buf_len > 4) {346 while (in_buf_len > 4) {
...@@ -341,80 +372,111 @@ pub const Base64DecoderUnsafe = struct {...@@ -341,80 +372,111 @@ pub const Base64DecoderUnsafe = struct {
341 }372 }
342};373};
343374
344fn calcDecodedSizeExactUnsafe(source: []const u8, pad_char: u8) usize {
345 if (source.len == 0) return 0;
346 var result = @divExact(source.len, 4) * 3;
347 if (source[source.len - 1] == pad_char) {
348 result -= 1;
349 if (source[source.len - 2] == pad_char) {
350 result -= 1;
351 }
352 }
353 return result;
354}
355
356test "base64" {375test "base64" {
357 @setEvalBranchQuota(8000);376 @setEvalBranchQuota(8000);
358 testBase64() catch unreachable;377 testBase64() catch unreachable;
359 comptime (testBase64() catch unreachable);378 comptime testAllApis(standard, "comptime", "Y29tcHRpbWU=") catch unreachable;
379}
380
381test "base64 url_safe" {
382 @setEvalBranchQuota(8000);
383 testBase64UrlSafe() catch unreachable;
384 comptime testAllApis(url_safe, "comptime", "Y29tcHRpbWU") catch unreachable;
360}385}
361386
362fn testBase64() !void {387fn testBase64() !void {
363 try testAllApis("", "");388 const codecs = standard;
364 try testAllApis("f", "Zg==");389
365 try testAllApis("fo", "Zm8=");390 try testAllApis(codecs, "", "");
366 try testAllApis("foo", "Zm9v");391 try testAllApis(codecs, "f", "Zg==");
367 try testAllApis("foob", "Zm9vYg==");392 try testAllApis(codecs, "fo", "Zm8=");
368 try testAllApis("fooba", "Zm9vYmE=");393 try testAllApis(codecs, "foo", "Zm9v");
369 try testAllApis("foobar", "Zm9vYmFy");394 try testAllApis(codecs, "foob", "Zm9vYg==");
370395 try testAllApis(codecs, "fooba", "Zm9vYmE=");
371 try testDecodeIgnoreSpace("", " ");396 try testAllApis(codecs, "foobar", "Zm9vYmFy");
372 try testDecodeIgnoreSpace("f", "Z g= =");397
373 try testDecodeIgnoreSpace("fo", " Zm8=");398 try testDecodeIgnoreSpace(codecs, "", " ");
374 try testDecodeIgnoreSpace("foo", "Zm9v ");399 try testDecodeIgnoreSpace(codecs, "f", "Z g= =");
375 try testDecodeIgnoreSpace("foob", "Zm9vYg = = ");400 try testDecodeIgnoreSpace(codecs, "fo", " Zm8=");
376 try testDecodeIgnoreSpace("fooba", "Zm9v YmE=");401 try testDecodeIgnoreSpace(codecs, "foo", "Zm9v ");
377 try testDecodeIgnoreSpace("foobar", " Z m 9 v Y m F y ");402 try testDecodeIgnoreSpace(codecs, "foob", "Zm9vYg = = ");
403 try testDecodeIgnoreSpace(codecs, "fooba", "Zm9v YmE=");
404 try testDecodeIgnoreSpace(codecs, "foobar", " Z m 9 v Y m F y ");
405
406 // test getting some api errors
407 try testError(codecs, "A", error.InvalidPadding);
408 try testError(codecs, "AA", error.InvalidPadding);
409 try testError(codecs, "AAA", error.InvalidPadding);
410 try testError(codecs, "A..A", error.InvalidCharacter);
411 try testError(codecs, "AA=A", error.InvalidPadding);
412 try testError(codecs, "AA/=", error.InvalidPadding);
413 try testError(codecs, "A/==", error.InvalidPadding);
414 try testError(codecs, "A===", error.InvalidPadding);
415 try testError(codecs, "====", error.InvalidPadding);
416
417 try testNoSpaceLeftError(codecs, "AA==");
418 try testNoSpaceLeftError(codecs, "AAA=");
419 try testNoSpaceLeftError(codecs, "AAAA");
420 try testNoSpaceLeftError(codecs, "AAAAAA==");
421}
422
423fn testBase64UrlSafe() !void {
424 const codecs = url_safe;
425
426 try testAllApis(codecs, "", "");
427 try testAllApis(codecs, "f", "Zg");
428 try testAllApis(codecs, "fo", "Zm8");
429 try testAllApis(codecs, "foo", "Zm9v");
430 try testAllApis(codecs, "foob", "Zm9vYg");
431 try testAllApis(codecs, "fooba", "Zm9vYmE");
432 try testAllApis(codecs, "foobar", "Zm9vYmFy");
433
434 try testDecodeIgnoreSpace(codecs, "", " ");
435 try testDecodeIgnoreSpace(codecs, "f", "Z g ");
436 try testDecodeIgnoreSpace(codecs, "fo", " Zm8");
437 try testDecodeIgnoreSpace(codecs, "foo", "Zm9v ");
438 try testDecodeIgnoreSpace(codecs, "foob", "Zm9vYg ");
439 try testDecodeIgnoreSpace(codecs, "fooba", "Zm9v YmE");
440 try testDecodeIgnoreSpace(codecs, "foobar", " Z m 9 v Y m F y ");
378441
379 // test getting some api errors442 // test getting some api errors
380 try testError("A", error.InvalidPadding);443 try testError(codecs, "A", error.InvalidPadding);
381 try testError("AA", error.InvalidPadding);444 try testError(codecs, "AAA=", error.InvalidCharacter);
382 try testError("AAA", error.InvalidPadding);445 try testError(codecs, "A..A", error.InvalidCharacter);
383 try testError("A..A", error.InvalidCharacter);446 try testError(codecs, "AA=A", error.InvalidCharacter);
384 try testError("AA=A", error.InvalidCharacter);447 try testError(codecs, "AA/=", error.InvalidCharacter);
385 try testError("AA/=", error.InvalidPadding);448 try testError(codecs, "A/==", error.InvalidCharacter);
386 try testError("A/==", error.InvalidPadding);449 try testError(codecs, "A===", error.InvalidCharacter);
387 try testError("A===", error.InvalidCharacter);450 try testError(codecs, "====", error.InvalidCharacter);
388 try testError("====", error.InvalidCharacter);451
389452 try testNoSpaceLeftError(codecs, "AA");
390 try testOutputTooSmallError("AA==");453 try testNoSpaceLeftError(codecs, "AAA");
391 try testOutputTooSmallError("AAA=");454 try testNoSpaceLeftError(codecs, "AAAA");
392 try testOutputTooSmallError("AAAA");455 try testNoSpaceLeftError(codecs, "AAAAAA");
393 try testOutputTooSmallError("AAAAAA==");
394}456}
395457
396fn testAllApis(expected_decoded: []const u8, expected_encoded: []const u8) !void {458fn testAllApis(codecs: Codecs, expected_decoded: []const u8, expected_encoded: []const u8) !void {
397 // Base64Encoder459 // Base64Encoder
398 {460 {
399 var buffer: [0x100]u8 = undefined;461 var buffer: [0x100]u8 = undefined;
400 const encoded = standard_encoder.encode(&buffer, expected_decoded);462 const encoded = codecs.Encoder.encode(&buffer, expected_decoded);
401 testing.expectEqualSlices(u8, expected_encoded, encoded);463 testing.expectEqualSlices(u8, expected_encoded, encoded);
402 }464 }
403465
404 // Base64Decoder466 // Base64Decoder
405 {467 {
406 var buffer: [0x100]u8 = undefined;468 var buffer: [0x100]u8 = undefined;
407 var decoded = buffer[0..try standard_decoder.calcSize(expected_encoded)];469 var decoded = buffer[0..try codecs.Decoder.calcSizeForSlice(expected_encoded)];
408 try standard_decoder.decode(decoded, expected_encoded);470 try codecs.Decoder.decode(decoded, expected_encoded);
409 testing.expectEqualSlices(u8, expected_decoded, decoded);471 testing.expectEqualSlices(u8, expected_decoded, decoded);
410 }472 }
411473
412 // Base64DecoderWithIgnore474 // Base64DecoderWithIgnore
413 {475 {
414 const standard_decoder_ignore_nothing = Base64DecoderWithIgnore.init(standard_alphabet_chars, standard_pad_char, "");476 const decoder_ignore_nothing = codecs.decoderWithIgnore("");
415 var buffer: [0x100]u8 = undefined;477 var buffer: [0x100]u8 = undefined;
416 var decoded = buffer[0..Base64DecoderWithIgnore.calcSizeUpperBound(expected_encoded.len)];478 var decoded = buffer[0..try decoder_ignore_nothing.calcSizeUpperBound(expected_encoded.len)];
417 var written = try standard_decoder_ignore_nothing.decode(decoded, expected_encoded);479 var written = try decoder_ignore_nothing.decode(decoded, expected_encoded);
418 testing.expect(written <= decoded.len);480 testing.expect(written <= decoded.len);
419 testing.expectEqualSlices(u8, expected_decoded, decoded[0..written]);481 testing.expectEqualSlices(u8, expected_decoded, decoded[0..written]);
420 }482 }
...@@ -422,40 +484,40 @@ fn testAllApis(expected_decoded: []const u8, expected_encoded: []const u8) !void...@@ -422,40 +484,40 @@ fn testAllApis(expected_decoded: []const u8, expected_encoded: []const u8) !void
422 // Base64DecoderUnsafe484 // Base64DecoderUnsafe
423 {485 {
424 var buffer: [0x100]u8 = undefined;486 var buffer: [0x100]u8 = undefined;
425 var decoded = buffer[0..standard_decoder_unsafe.calcSize(expected_encoded)];487 var decoded = buffer[0..try codecs.DecoderUnsafe.calcSizeForSlice(expected_encoded)];
426 standard_decoder_unsafe.decode(decoded, expected_encoded);488 codecs.DecoderUnsafe.decode(decoded, expected_encoded);
427 testing.expectEqualSlices(u8, expected_decoded, decoded);489 testing.expectEqualSlices(u8, expected_decoded, decoded);
428 }490 }
429}491}
430492
431fn testDecodeIgnoreSpace(expected_decoded: []const u8, encoded: []const u8) !void {493fn testDecodeIgnoreSpace(codecs: Codecs, expected_decoded: []const u8, encoded: []const u8) !void {
432 const standard_decoder_ignore_space = Base64DecoderWithIgnore.init(standard_alphabet_chars, standard_pad_char, " ");494 const decoder_ignore_space = codecs.decoderWithIgnore(" ");
433 var buffer: [0x100]u8 = undefined;495 var buffer: [0x100]u8 = undefined;
434 var decoded = buffer[0..Base64DecoderWithIgnore.calcSizeUpperBound(encoded.len)];496 var decoded = buffer[0..try decoder_ignore_space.calcSizeUpperBound(encoded.len)];
435 var written = try standard_decoder_ignore_space.decode(decoded, encoded);497 var written = try decoder_ignore_space.decode(decoded, encoded);
436 testing.expectEqualSlices(u8, expected_decoded, decoded[0..written]);498 testing.expectEqualSlices(u8, expected_decoded, decoded[0..written]);
437}499}
438500
439fn testError(encoded: []const u8, expected_err: anyerror) !void {501fn testError(codecs: Codecs, encoded: []const u8, expected_err: anyerror) !void {
440 const standard_decoder_ignore_space = Base64DecoderWithIgnore.init(standard_alphabet_chars, standard_pad_char, " ");502 const decoder_ignore_space = codecs.decoderWithIgnore(" ");
441 var buffer: [0x100]u8 = undefined;503 var buffer: [0x100]u8 = undefined;
442 if (standard_decoder.calcSize(encoded)) |decoded_size| {504 if (codecs.Decoder.calcSizeForSlice(encoded)) |decoded_size| {
443 var decoded = buffer[0..decoded_size];505 var decoded = buffer[0..decoded_size];
444 if (standard_decoder.decode(decoded, encoded)) |_| {506 if (codecs.Decoder.decode(decoded, encoded)) |_| {
445 return error.ExpectedError;507 return error.ExpectedError;
446 } else |err| if (err != expected_err) return err;508 } else |err| if (err != expected_err) return err;
447 } else |err| if (err != expected_err) return err;509 } else |err| if (err != expected_err) return err;
448510
449 if (standard_decoder_ignore_space.decode(buffer[0..], encoded)) |_| {511 if (decoder_ignore_space.decode(buffer[0..], encoded)) |_| {
450 return error.ExpectedError;512 return error.ExpectedError;
451 } else |err| if (err != expected_err) return err;513 } else |err| if (err != expected_err) return err;
452}514}
453515
454fn testOutputTooSmallError(encoded: []const u8) !void {516fn testNoSpaceLeftError(codecs: Codecs, encoded: []const u8) !void {
455 const standard_decoder_ignore_space = Base64DecoderWithIgnore.init(standard_alphabet_chars, standard_pad_char, " ");517 const decoder_ignore_space = codecs.decoderWithIgnore(" ");
456 var buffer: [0x100]u8 = undefined;518 var buffer: [0x100]u8 = undefined;
457 var decoded = buffer[0 .. calcDecodedSizeExactUnsafe(encoded, standard_pad_char) - 1];519 var decoded = buffer[0 .. (try codecs.Decoder.calcSizeForSlice(encoded)) - 1];
458 if (standard_decoder_ignore_space.decode(decoded, encoded)) |_| {520 if (decoder_ignore_space.decode(decoded, encoded)) |_| {
459 return error.ExpectedError;521 return error.ExpectedError;
460 } else |err| if (err != error.OutputTooSmall) return err;522 } else |err| if (err != error.NoSpaceLeft) return err;
461}523}
lib/std/fs.zig+5-5
...@@ -50,13 +50,13 @@ pub const MAX_PATH_BYTES = switch (builtin.os.tag) {...@@ -50,13 +50,13 @@ pub const MAX_PATH_BYTES = switch (builtin.os.tag) {
50 else => @compileError("Unsupported OS"),50 else => @compileError("Unsupported OS"),
51};51};
5252
53pub const base64_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";53pub const base64_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".*;
5454
55/// Base64 encoder, replacing the standard `+/` with `-_` so that it can be used in a file name on any filesystem.55/// Base64 encoder, replacing the standard `+/` with `-_` so that it can be used in a file name on any filesystem.
56pub const base64_encoder = base64.Base64Encoder.init(base64_alphabet, base64.standard_pad_char);56pub const base64_encoder = base64.Base64Encoder.init(base64_alphabet, null);
5757
58/// Base64 decoder, replacing the standard `+/` with `-_` so that it can be used in a file name on any filesystem.58/// Base64 decoder, replacing the standard `+/` with `-_` so that it can be used in a file name on any filesystem.
59pub const base64_decoder = base64.Base64Decoder.init(base64_alphabet, base64.standard_pad_char);59pub const base64_decoder = base64.Base64Decoder.init(base64_alphabet, null);
6060
61/// Whether or not async file system syscalls need a dedicated thread because the operating61/// Whether or not async file system syscalls need a dedicated thread because the operating
62/// system does not support non-blocking I/O on the file system.62/// system does not support non-blocking I/O on the file system.
...@@ -77,7 +77,7 @@ pub fn atomicSymLink(allocator: *Allocator, existing_path: []const u8, new_path:...@@ -77,7 +77,7 @@ pub fn atomicSymLink(allocator: *Allocator, existing_path: []const u8, new_path:
77 const dirname = path.dirname(new_path) orelse ".";77 const dirname = path.dirname(new_path) orelse ".";
7878
79 var rand_buf: [AtomicFile.RANDOM_BYTES]u8 = undefined;79 var rand_buf: [AtomicFile.RANDOM_BYTES]u8 = undefined;
80 const tmp_path = try allocator.alloc(u8, dirname.len + 1 + base64.Base64Encoder.calcSize(rand_buf.len));80 const tmp_path = try allocator.alloc(u8, dirname.len + 1 + base64_encoder.calcSize(rand_buf.len));
81 defer allocator.free(tmp_path);81 defer allocator.free(tmp_path);
82 mem.copy(u8, tmp_path[0..], dirname);82 mem.copy(u8, tmp_path[0..], dirname);
83 tmp_path[dirname.len] = path.sep;83 tmp_path[dirname.len] = path.sep;
...@@ -142,7 +142,7 @@ pub const AtomicFile = struct {...@@ -142,7 +142,7 @@ pub const AtomicFile = struct {
142 const InitError = File.OpenError;142 const InitError = File.OpenError;
143143
144 const RANDOM_BYTES = 12;144 const RANDOM_BYTES = 12;
145 const TMP_PATH_LEN = base64.Base64Encoder.calcSize(RANDOM_BYTES);145 const TMP_PATH_LEN = base64_encoder.calcSize(RANDOM_BYTES);
146146
147 /// Note that the `Dir.atomicFile` API may be more handy than this lower-level function.147 /// Note that the `Dir.atomicFile` API may be more handy than this lower-level function.
148 pub fn init(148 pub fn init(
lib/std/testing.zig+1-1
...@@ -298,7 +298,7 @@ pub const TmpDir = struct {...@@ -298,7 +298,7 @@ pub const TmpDir = struct {
298 sub_path: [sub_path_len]u8,298 sub_path: [sub_path_len]u8,
299299
300 const random_bytes_count = 12;300 const random_bytes_count = 12;
301 const sub_path_len = std.base64.Base64Encoder.calcSize(random_bytes_count);301 const sub_path_len = std.fs.base64_encoder.calcSize(random_bytes_count);
302302
303 pub fn cleanup(self: *TmpDir) void {303 pub fn cleanup(self: *TmpDir) void {
304 self.dir.close();304 self.dir.close();
test/standalone/mix_o_files/base64.zig+2-2
...@@ -3,8 +3,8 @@ const base64 = @import("std").base64;...@@ -3,8 +3,8 @@ const base64 = @import("std").base64;
3export fn decode_base_64(dest_ptr: [*]u8, dest_len: usize, source_ptr: [*]const u8, source_len: usize) usize {3export fn decode_base_64(dest_ptr: [*]u8, dest_len: usize, source_ptr: [*]const u8, source_len: usize) usize {
4 const src = source_ptr[0..source_len];4 const src = source_ptr[0..source_len];
5 const dest = dest_ptr[0..dest_len];5 const dest = dest_ptr[0..dest_len];
6 const base64_decoder = base64.standard_decoder_unsafe;6 const base64_decoder = base64.standard.DecoderUnsafe;
7 const decoded_size = base64_decoder.calcSize(src);7 const decoded_size = base64_decoder.calcSizeForSlice(src) catch unreachable;
8 base64_decoder.decode(dest[0..decoded_size], src);8 base64_decoder.decode(dest[0..decoded_size], src);
9 return decoded_size;9 return decoded_size;
10}10}