CH-J Toolsserverhåndtering over SSH
Menu
Offentliggjort kilde

CH-J Tools

Gennemse mapper og filer til en specifik applikationsudgivelse.

Download kilde ZIP
CH-J / Server Manager 1704 filer
0.1.2-beta Tilføjet: · Bygget:
engine.rs Tilføjet: Dato ikke registreret 20,9 KB · 461 linjer
Kode Download fil
1//! Hash engine (replaces the 0.1.0 `hashWorker.js`): one sequential read for every streaming
2//! algorithm, then the random-access City/Farm hashes over a cached view of the same file.
3//! Runs on a blocking thread; cancellation is checked between chunks and cache refills.
5use super::algorithms::Request;
6use super::city::{self, View};
7use super::portable::{Fnv, Murmur128x64, Murmur128x86, Murmur32, SipHash24};
8use crate::core::error::{CoreError, CoreResult};
9use digest::{DynDigest, ExtendableOutput, Update, XofReader};
10use highway::HighwayHash;
11use std::fs::File;
12use std::io::{Read, Seek, SeekFrom};
13use std::path::Path;
14use std::sync::atomic::{AtomicBool, Ordering};
16static CRC16: crc::Crc<u16> = crc::Crc::<u16>::new(&crc::CRC_16_IBM_3740);
17static CRC32: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
18static CRC32C: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISCSI);
19static CRC64_ECMA: crc::Crc<u64> = crc::Crc::<u64>::new(&crc::CRC_64_ECMA_182);
20static CRC64_XZ: crc::Crc<u64> = crc::Crc::<u64>::new(&crc::CRC_64_XZ);
22pub fn cancelled() -> CoreError {
23 CoreError::new("HASH_CANCELLED", "Hash job was cancelled.")
26fn changed(message: &str) -> CoreError {
27 CoreError::new("HASH_FILE_CHANGED", message)
30enum Stream {
31 Digest(Box<dyn DynDigest + Send>),
32 Shake128(sha3::Shake128, usize),
33 Shake256(sha3::Shake256, usize),
34 K12(Box<k12::KangarooTwelve<'static>>, usize),
35 Blake3(Box<blake3::Hasher>),
36 Xxh32(xxhash_rust::xxh32::Xxh32),
37 Xxh64(xxhash_rust::xxh64::Xxh64),
38 Xxh3(Box<xxhash_rust::xxh3::Xxh3>, bool),
39 Highway(highway::HighwayHasher, u32),
40 Crc16(crc::Digest<'static, u16>),
41 Crc32(crc::Digest<'static, u32>),
42 Crc64(crc::Digest<'static, u64>),
43 Adler(adler::Adler32),
44 Murmur32(Murmur32),
45 Murmur128x86(Murmur128x86),
46 Murmur128x64(Murmur128x64),
47 Sip(SipHash24),
48 Fnv(Fnv),
51impl Stream {
52 fn new(request: &Request) -> Option<Stream> {
53 let seed = request.seed.unwrap_or(0);
54 let digest = |d: Box<dyn DynDigest + Send>| Some(Stream::Digest(d));
55 match request.id {
56 "sha224" => digest(Box::new(sha2::Sha224::default())),
57 "sha256" => digest(Box::new(sha2::Sha256::default())),
58 "sha384" => digest(Box::new(sha2::Sha384::default())),
59 "sha512" => digest(Box::new(sha2::Sha512::default())),
60 "sha512-224" => digest(Box::new(sha2::Sha512_224::default())),
61 "sha512-256" => digest(Box::new(sha2::Sha512_256::default())),
62 "sha3-224" => digest(Box::new(sha3::Sha3_224::default())),
63 "sha3-256" => digest(Box::new(sha3::Sha3_256::default())),
64 "sha3-384" => digest(Box::new(sha3::Sha3_384::default())),
65 "sha3-512" => digest(Box::new(sha3::Sha3_512::default())),
66 "blake2b-512" => digest(Box::new(blake2::Blake2b512::default())),
67 "blake2s-256" => digest(Box::new(blake2::Blake2s256::default())),
68 "ripemd160" => digest(Box::new(ripemd::Ripemd160::default())),
69 "whirlpool" => digest(Box::new(whirlpool::Whirlpool::default())),
70 "tiger" => digest(Box::new(tiger::Tiger::default())),
71 "tiger2" => digest(Box::new(tiger::Tiger2::default())),
72 "md5" => digest(Box::new(md5::Md5::default())),
73 "sha1" => digest(Box::new(sha1::Sha1::default())),
74 "shake128" => Some(Stream::Shake128(Default::default(), request.output_bytes.unwrap_or(32))),
75 "shake256" => Some(Stream::Shake256(Default::default(), request.output_bytes.unwrap_or(64))),
76 "kangaroo-twelve" => Some(Stream::K12(Box::new(k12::KangarooTwelve::default()), request.output_bytes.unwrap_or(32))),
77 "blake3" => Some(Stream::Blake3(Box::new(blake3::Hasher::new()))),
78 "xxh32" => Some(Stream::Xxh32(xxhash_rust::xxh32::Xxh32::new(seed as u32))),
79 "xxh64" => Some(Stream::Xxh64(xxhash_rust::xxh64::Xxh64::new(seed))),
80 "xxh3-64" => Some(Stream::Xxh3(Box::new(xxhash_rust::xxh3::Xxh3::with_seed(seed)), false)),
81 "xxh3-128" => Some(Stream::Xxh3(Box::new(xxhash_rust::xxh3::Xxh3::with_seed(seed)), true)),
82 "highwayhash64" | "highwayhash128" | "highwayhash256" => {
83 let key = request.key.as_deref().unwrap_or(&[0; 32]);
84 let words: [u64; 4] = std::array::from_fn(|i| u64::from_le_bytes(key[i * 8..i * 8 + 8].try_into().unwrap()));
85 let bits = match request.id { "highwayhash64" => 64, "highwayhash128" => 128, _ => 256 };
86 Some(Stream::Highway(highway::HighwayHasher::new(highway::Key(words)), bits))
87 }
88 "crc16-ccitt-false" => Some(Stream::Crc16(CRC16.digest())),
89 "crc32" => Some(Stream::Crc32(CRC32.digest())),
90 "crc32c" => Some(Stream::Crc32(CRC32C.digest())),
91 "crc64-ecma" => Some(Stream::Crc64(CRC64_ECMA.digest())),
92 "crc64-xz" => Some(Stream::Crc64(CRC64_XZ.digest())),
93 "adler32" => Some(Stream::Adler(adler::Adler32::new())),
94 "murmur3-x86-32" => Some(Stream::Murmur32(Murmur32::new(seed as u32))),
95 "murmur3-x86-128" => Some(Stream::Murmur128x86(Murmur128x86::new(seed as u32))),
96 "murmur3-x64-128" => Some(Stream::Murmur128x64(Murmur128x64::new(seed))),
97 "siphash-2-4" => Some(Stream::Sip(SipHash24::new(request.key.as_deref().unwrap_or(&[0; 16])))),
98 "fnv1-32" => Some(Stream::Fnv(Fnv::new(32, false))),
99 "fnv1-64" => Some(Stream::Fnv(Fnv::new(64, false))),
100 "fnv1a-32" => Some(Stream::Fnv(Fnv::new(32, true))),
101 "fnv1a-64" => Some(Stream::Fnv(Fnv::new(64, true))),
102 _ => None,
103 }
104 }
106 fn update(&mut self, data: &[u8]) {
107 match self {
108 Stream::Digest(d) => d.update(data),
109 Stream::Shake128(h, _) => Update::update(h, data),
110 Stream::Shake256(h, _) => Update::update(h, data),
111 Stream::K12(h, _) => Update::update(h.as_mut(), data),
112 Stream::Blake3(h) => {
113 h.update(data);
114 }
115 Stream::Xxh32(h) => h.update(data),
116 Stream::Xxh64(h) => h.update(data),
117 Stream::Xxh3(h, _) => h.update(data),
118 Stream::Highway(h, _) => h.append(data),
119 Stream::Crc16(h) => h.update(data),
120 Stream::Crc32(h) => h.update(data),
121 Stream::Crc64(h) => h.update(data),
122 Stream::Adler(h) => h.write_slice(data),
123 Stream::Murmur32(h) => h.update(data),
124 Stream::Murmur128x86(h) => h.update(data),
125 Stream::Murmur128x64(h) => h.update(data),
126 Stream::Sip(h) => h.update(data),
127 Stream::Fnv(h) => h.update(data),
128 }
129 }
131 fn finish(self) -> Vec<u8> {
132 fn xof(reader: &mut dyn XofReader, n: usize) -> Vec<u8> {
133 let mut out = vec![0u8; n];
134 reader.read(&mut out);
135 out
136 }
137 match self {
138 Stream::Digest(d) => d.finalize().to_vec(),
139 Stream::Shake128(h, n) => xof(&mut h.finalize_xof(), n),
140 Stream::Shake256(h, n) => xof(&mut h.finalize_xof(), n),
141 Stream::K12(h, n) => xof(&mut h.finalize_xof(), n),
142 Stream::Blake3(h) => h.finalize().as_bytes().to_vec(),
143 Stream::Xxh32(h) => h.digest().to_be_bytes().to_vec(),
144 Stream::Xxh64(h) => h.digest().to_be_bytes().to_vec(),
145 Stream::Xxh3(h, wide) => if wide { h.digest128().to_be_bytes().to_vec() } else { h.digest().to_be_bytes().to_vec() },
146 Stream::Highway(h, bits) => match bits {
147 64 => h.finalize64().to_le_bytes().to_vec(), // 0.1.0 (highwayhasher) byte order
148 128 => h.finalize128().iter().flat_map(|w| w.to_le_bytes()).collect(),
149 _ => h.finalize256().iter().flat_map(|w| w.to_le_bytes()).collect(),
150 },
151 Stream::Crc16(h) => h.finalize().to_be_bytes().to_vec(),
152 Stream::Crc32(h) => h.finalize().to_be_bytes().to_vec(),
153 Stream::Crc64(h) => h.finalize().to_be_bytes().to_vec(),
154 Stream::Adler(h) => h.checksum().to_be_bytes().to_vec(),
155 Stream::Murmur32(h) => h.finish(),
156 Stream::Murmur128x86(h) => h.finish(),
157 Stream::Murmur128x64(h) => h.finish(),
158 Stream::Sip(h) => h.finish(),
159 Stream::Fnv(h) => h.finish(),
160 }
161 }
164fn is_random_access(id: &str) -> bool {
165 matches!(id, "cityhash32" | "cityhash64" | "cityhash128" | "farmhash32" | "farmhash64" | "farmhash128")
168fn random_access(requests: &[Request], view: &mut dyn View) -> Vec<(&'static str, Vec<u8>)> {
169 let mut out = Vec::new();
170 let mut wide: Option<Vec<u8>> = None;
171 for request in requests.iter().filter(|r| is_random_access(r.id)) {
172 let digest = match request.id {
173 "cityhash32" => city::city32(view).to_be_bytes().to_vec(),
174 "cityhash64" => city::city64(view).to_be_bytes().to_vec(),
175 "farmhash32" => city::farm32(view).to_be_bytes().to_vec(),
176 "farmhash64" => city::farm64(view).to_be_bytes().to_vec(),
177 // FarmHash Fingerprint128 is CityHash128 by definition (0.1.0 computed it once).
178 _ => wide.get_or_insert_with(|| city::digest128(city::city128(view))).clone(),
179 };
180 out.push((request.id, digest));
181 }
182 out
185/// Pure in-memory digests (tests, small inputs).
186pub fn digest_bytes(requests: &[Request], data: &[u8]) -> Vec<(&'static str, Vec<u8>)> {
187 let mut streams: Vec<(&'static str, Stream)> = requests.iter().filter_map(|r| Stream::new(r).map(|s| (r.id, s))).collect();
188 for (_, stream) in streams.iter_mut() {
189 stream.update(data);
190 }
191 let mut results: Vec<(&'static str, Vec<u8>)> = streams.into_iter().map(|(id, s)| (id, s.finish())).collect();
192 results.extend(random_access(requests, &mut city::SliceView(data)));
193 order(requests, results)
196fn order(requests: &[Request], results: Vec<(&'static str, Vec<u8>)>) -> Vec<(&'static str, Vec<u8>)> {
197 requests.iter().filter_map(|r| results.iter().find(|(id, _)| *id == r.id).cloned()).collect()
200/// File identity captured when a file is authorized (0.1.0: dev, ino, size, mtimeNs).
201#[derive(Debug, Clone, PartialEq, Eq)]
202pub struct Identity {
203 pub size: u64,
204 pub modified_ns: i128,
205 pub device: u64,
206 pub inode: u64,
209impl Identity {
210 pub fn of(meta: &std::fs::Metadata) -> Identity {
211 let modified_ns = meta
212 .modified()
213 .ok()
214 .map(|t| match t.duration_since(std::time::UNIX_EPOCH) {
215 Ok(d) => d.as_nanos() as i128,
216 Err(e) => -(e.duration().as_nanos() as i128),
217 })
218 .unwrap_or(0);
219 #[cfg(unix)]
220 let (device, inode) = {
221 use std::os::unix::fs::MetadataExt;
222 (meta.dev(), meta.ino())
223 };
224 #[cfg(windows)]
225 let (device, inode) = {
226 use std::os::windows::fs::MetadataExt;
227 (0u64, meta.creation_time())
228 };
229 Identity { size: meta.len(), modified_ns, device, inode }
230 }
233/// Opens a regular file without following a final symbolic link.
234pub fn open_no_follow(path: &Path) -> std::io::Result<File> {
235 let mut options = std::fs::OpenOptions::new();
236 options.read(true);
237 #[cfg(unix)]
238 {
239 use std::os::unix::fs::OpenOptionsExt;
240 options.custom_flags(libc::O_NOFOLLOW);
241 }
242 #[cfg(windows)]
243 {
244 use std::os::windows::fs::OpenOptionsExt;
245 options.custom_flags(0x0020_0000); // FILE_FLAG_OPEN_REPARSE_POINT
246 }
247 options.open(path)
250/// Lazily cached file view for City/Farm (reads 1 MiB windows; checks cancellation).
251struct FileView<'a> {
252 file: &'a mut File,
253 len: usize,
254 cache: Vec<u8>,
255 start: usize,
256 cancel: &'a AtomicBool,
257 error: Option<CoreError>,
260const WINDOW: usize = 1024 * 1024;
262impl FileView<'_> {
263 fn ensure(&mut self, offset: usize, width: usize) -> Option<usize> {
264 if self.error.is_some() {
265 return None;
266 }
267 if offset >= self.start && offset + width <= self.start + self.cache.len() {
268 return Some(offset - self.start);
269 }
270 if self.cancel.load(Ordering::Relaxed) {
271 self.error = Some(cancelled());
272 return None;
273 }
274 let start = (offset / WINDOW) * WINDOW;
275 let bytes = (WINDOW + 16).min(self.len - start);
276 self.cache.resize(bytes, 0);
277 let result = self.file.seek(SeekFrom::Start(start as u64)).and_then(|_| self.file.read_exact(&mut self.cache));
278 if result.is_err() {
279 self.error = Some(changed("Selected file ended unexpectedly."));
280 self.cache.clear();
281 return None;
282 }
283 self.start = start;
284 Some(offset - start)
285 }
288impl View for FileView<'_> {
289 fn len(&self) -> usize {
290 self.len
291 }
292 fn byte(&mut self, offset: usize) -> u8 {
293 self.ensure(offset, 1).map(|i| self.cache[i]).unwrap_or(0)
294 }
295 fn u32(&mut self, offset: usize) -> u32 {
296 self.ensure(offset, 4).map(|i| u32::from_le_bytes(self.cache[i..i + 4].try_into().unwrap())).unwrap_or(0)
297 }
298 fn u64(&mut self, offset: usize) -> u64 {
299 self.ensure(offset, 8).map(|i| u64::from_le_bytes(self.cache[i..i + 8].try_into().unwrap())).unwrap_or(0)
300 }
303pub const CHUNK: usize = 1024 * 1024;
305/// Hashes one file. `progress(bytes_read)` is called at most every 100 ms and at the end.
306pub fn hash_file(path: &Path, expected: &Identity, requests: &[Request], cancel: &AtomicBool, mut progress: impl FnMut(u64)) -> CoreResult<Vec<(&'static str, Vec<u8>)>> {
307 let mut file = open_no_follow(path).map_err(super::service::io_error)?;
308 let before = file.metadata().map_err(super::service::io_error)?;
309 if !before.is_file() {
310 return Err(CoreError::new("HASH_NOT_REGULAR_FILE", "Only regular files can be hashed."));
311 }
312 if Identity::of(&before) != *expected {
313 return Err(changed("The selected file changed before hashing started."));
314 }
315 let size = before.len();
316 let mut streams: Vec<(&'static str, Stream)> = requests.iter().filter(|r| !is_random_access(r.id)).filter_map(|r| Stream::new(r).map(|s| (r.id, s))).collect();
317 let mut buffer = vec![0u8; CHUNK];
318 let mut position = 0u64;
319 let mut last = std::time::Instant::now() - std::time::Duration::from_secs(1);
320 while position < size {
321 if cancel.load(Ordering::Relaxed) {
322 return Err(cancelled());
323 }
324 let want = ((size - position) as usize).min(buffer.len());
325 let read = file.read(&mut buffer[..want]).map_err(super::service::io_error)?;
326 if read == 0 {
327 return Err(changed("Selected file ended unexpectedly."));
328 }
329 for (_, stream) in streams.iter_mut() {
330 stream.update(&buffer[..read]);
331 }
332 position += read as u64;
333 if last.elapsed() >= std::time::Duration::from_millis(100) || position == size {
334 progress(position);
335 last = std::time::Instant::now();
336 }
337 }
338 if size == 0 {
339 progress(0);
340 }
341 let mut results: Vec<(&'static str, Vec<u8>)> = streams.into_iter().map(|(id, s)| (id, s.finish())).collect();
342 if requests.iter().any(|r| is_random_access(r.id)) {
343 if size > usize::MAX as u64 {
344 return Err(CoreError::new("HASH_FILE_TOO_LARGE", "Selected file exceeds the supported size."));
345 }
346 let mut view = FileView { file: &mut file, len: size as usize, cache: Vec::new(), start: 0, cancel, error: None };
347 let digests = random_access(requests, &mut view);
348 if let Some(error) = view.error.take() {
349 return Err(error);
350 }
351 results.extend(digests);
352 }
353 if cancel.load(Ordering::Relaxed) {
354 return Err(cancelled());
355 }
356 let after = file.metadata().map_err(super::service::io_error)?;
357 if Identity::of(&after) != *expected {
358 return Err(changed("The file changed while it was being hashed."));
359 }
360 Ok(order(requests, results))
363#[cfg(test)]
364mod tests {
365 use super::super::algorithms::{normalize_requests, ALGORITHMS};
366 use super::*;
367 use serde_json::{json, Value};
369 fn data(n: usize) -> Vec<u8> {
370 (0..n).map(|i| (((i * 31 + 7) ^ (i >> 8) ^ (i >> 3)) & 0xff) as u8).collect()
371 }
373 /// Bit-exact parity with the 0.1.0 Electron worker (vectors generated from its code).
374 #[test]
375 fn parity_with_0_1_0_worker() {
376 let fixture: Value = serde_json::from_str(include_str!("../../../tests/fixtures/hash/parity-0.1.0.json")).unwrap();
377 let variants: Vec<Vec<Request>> = fixture["variants"].as_array().unwrap().iter().map(|v| normalize_requests(v).unwrap()).collect();
378 let mut failures: Vec<String> = Vec::new();
379 for case in fixture["cases"].as_array().unwrap() {
380 let length = case["length"].as_u64().unwrap() as usize;
381 let requests = &variants[case["variant"].as_u64().unwrap() as usize];
382 let input = data(length);
383 for (id, digest) in digest_bytes(requests, &input) {
384 // 0.1.0 MurmurHash3 x86-128 mixed the block state in the wrong order (non-standard
385 // for inputs of 16+ bytes); 0.1.1 uses the reference algorithm (see test below).
386 if id == "murmur3-x86-128" && length >= 16 {
387 continue;
388 }
389 let expected = case["digests"][id].as_str().unwrap();
390 if hex::encode(&digest) != expected {
391 failures.push(format!("{id} len={length} variant={}", case["variant"]));
392 }
393 }
394 }
395 let mut by_algorithm: std::collections::BTreeMap<String, usize> = Default::default();
396 for f in &failures {
397 *by_algorithm.entry(f.split(' ').next().unwrap().to_string()).or_default() += 1;
398 }
399 assert!(failures.is_empty(), "{} mismatches: {:?}; first: {:?}", failures.len(), by_algorithm, failures.iter().take(5).collect::<Vec<_>>());
400 }
402 /// Published reference vectors (FIPS/RFC/upstream suites) shipped since 0.1.0.
403 #[test]
404 fn standard_vectors() {
405 let fixture: Value = serde_json::from_str(include_str!("../../../../tools/test-fixtures/hash-vectors.json")).unwrap();
406 let mut failures = Vec::new();
407 for (name, message) in fixture["messages"].as_object().unwrap() {
408 let vectors = fixture["vectors"][name].as_object().unwrap();
409 let ids: Vec<Value> = vectors.keys().filter(|id| super::super::algorithms::by_id(id).is_some()).map(|id| json!(id)).collect();
410 let mut requests = normalize_requests(&Value::Array(ids)).unwrap();
411 for r in requests.iter_mut() {
412 if r.key.is_some() {
413 let n = r.key.as_ref().unwrap().len();
414 r.key = Some((0..n as u8).collect());
415 }
416 }
417 for (id, digest) in digest_bytes(&requests, message.as_str().unwrap().as_bytes()) {
418 if hex::encode(&digest) != vectors[id].as_str().unwrap() {
419 failures.push(format!("{id}/{name}"));
420 }
421 }
422 }
423 assert!(failures.is_empty(), "{failures:?}");
424 }
426 /// MurmurHash3 x86-128 against the independent `murmur3` crate (reference C++ port).
427 #[test]
428 fn murmur_x86_128_matches_reference() {
429 for (length, seed) in [(0usize, 0u32), (3, 7), (15, 0), (16, 0), (17, 1), (40, 0x9e3779b9), (1000, 42)] {
430 let input = data(length);
431 let reference = murmur3::murmur3_x86_128(&mut std::io::Cursor::new(&input), seed).unwrap();
432 let mut ours = Murmur128x86::new(seed);
433 ours.update(&input);
434 // The crate packs h1 into the low word; 0.1.0/0.1.1 print h1..h4 big-endian.
435 let words = [reference as u32, (reference >> 32) as u32, (reference >> 64) as u32, (reference >> 96) as u32];
436 let expected: Vec<u8> = words.iter().flat_map(|w| w.to_be_bytes()).collect();
437 assert_eq!(ours.finish(), expected, "length {length}");
438 }
439 }
441 #[test]
442 fn file_hash_matches_memory_and_detects_cancel() {
443 let dir = tempfile::tempdir().unwrap();
444 let path = dir.path().join("f.bin");
445 let input = data(3 * CHUNK + 12345);
446 std::fs::write(&path, &input).unwrap();
447 let all: Vec<Value> = ALGORITHMS.iter().map(|a| json!(a.id)).collect();
448 let requests = normalize_requests(&Value::Array(all)).unwrap();
449 let identity = Identity::of(&std::fs::metadata(&path).unwrap());
450 let cancel = AtomicBool::new(false);
451 let mut seen = 0;
452 let from_file = hash_file(&path, &identity, &requests, &cancel, |b| seen = b).unwrap();
453 assert_eq!(seen, input.len() as u64);
454 assert_eq!(from_file, digest_bytes(&requests, &input));
455 cancel.store(true, Ordering::Relaxed);
456 assert_eq!(hash_file(&path, &identity, &requests, &cancel, |_| {}).unwrap_err().code, "HASH_CANCELLED");
457 let stale = Identity { size: 1, ..identity };
458 assert_eq!(hash_file(&path, &stale, &requests, &AtomicBool::new(false), |_| {}).unwrap_err().code, "HASH_FILE_CHANGED");
459 }
SHA-256bf06dbb7dc418efc9652a5415527b395e618b817a06a1259d64c562b952b1bbe

CH-J Proprietary Software License 1.14 · Kilden leveres under CH-J Proprietary Software License 1.14. Dens tilgængelighed ændrer ikke licensbetingelserne eller giver yderligere rettigheder.

Arkiv SHA-256ac8fe790f4ba60fc098f8c985fc2656cfcc6cdc21fb34b30e308a2b54e62258f
Filer