diff options
-rw-r--r-- | .gitignore | 6 | ||||
-rw-r--r-- | Cargo.toml | 53 | ||||
-rw-r--r-- | LICENSE_APACHE | 176 | ||||
-rw-r--r-- | LICENSE_MIT | 25 | ||||
-rw-r--r-- | README.md | 1 | ||||
-rw-r--r-- | data/0.png | bin | 0 -> 402 bytes | |||
-rw-r--r-- | data/1.png | bin | 0 -> 235 bytes | |||
-rw-r--r-- | data/2.png | bin | 0 -> 326 bytes | |||
-rw-r--r-- | data/2021-12-03 18-22-11.png | bin | 0 -> 1860275 bytes | |||
-rw-r--r-- | data/3.png | bin | 0 -> 368 bytes | |||
-rw-r--r-- | data/4.png | bin | 0 -> 331 bytes | |||
-rw-r--r-- | data/5.png | bin | 0 -> 357 bytes | |||
-rw-r--r-- | data/6.png | bin | 0 -> 397 bytes | |||
-rw-r--r-- | data/7.png | bin | 0 -> 337 bytes | |||
-rw-r--r-- | data/8.png | bin | 0 -> 386 bytes | |||
-rw-r--r-- | data/9.png | bin | 0 -> 373 bytes | |||
-rw-r--r-- | data/test.png | bin | 0 -> 1567 bytes | |||
-rw-r--r-- | index.html | 91 | ||||
-rw-r--r-- | src/lib.rs | 158 | ||||
-rw-r--r-- | src/ocr.rs | 50 | ||||
-rw-r--r-- | tests/web.rs | 13 |
21 files changed, 573 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4e30131 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/target +**/*.rs.bk +Cargo.lock +bin/ +pkg/ +wasm-pack.log diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..767c8b8 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "queue-go-brrr" +version = "0.1.0" +authors = ["Melody Horn <melody@boringcactus.com>"] +edition = "2018" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +base64 = "0.13.0" +gloo = "0.4.0" +image = "0.23.14" +imageproc = "0.22.0" +js-sys = "0.3.55" +wasm-bindgen = "0.2.63" +wasm-bindgen-futures = "0.4.28" +web-sys = { version = "0.3.55", features = [ + "Window", + "Navigator", + "MediaDevices", + "DisplayMediaStreamConstraints", + "MediaStream", + "MediaStreamTrack", + "Document", + "Element", + "HtmlVideoElement", + "HtmlMediaElement", + "EventTarget", + "HtmlCanvasElement", + "CanvasRenderingContext2d", + "Blob", +] } + +# The `console_error_panic_hook` crate provides better debugging of panics by +# logging them with `console.error`. This is great for development, but requires +# all the `std::fmt` and `std::panicking` infrastructure, so isn't great for +# code size when deploying. +console_error_panic_hook = "0.1.6" + +# `wee_alloc` is a tiny allocator for wasm that is only ~1K in code size +# compared to the default allocator's ~10K. It is slower than the default +# allocator, however. +# +# Unfortunately, `wee_alloc` requires nightly Rust when targeting wasm for now. +wee_alloc = { version = "0.4.5", optional = true } + +[dev-dependencies] +wasm-bindgen-test = "0.3.13" + +[profile.release] +# Tell `rustc` to optimize for small code size. +opt-level = "s" diff --git a/LICENSE_APACHE b/LICENSE_APACHE new file mode 100644 index 0000000..1b5ec8b --- /dev/null +++ b/LICENSE_APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/LICENSE_MIT b/LICENSE_MIT new file mode 100644 index 0000000..58b622c --- /dev/null +++ b/LICENSE_MIT @@ -0,0 +1,25 @@ +Copyright (c) 2018 Melody Horn <melody@boringcactus.com> + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..161cd8e --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +penis diff --git a/data/0.png b/data/0.png Binary files differnew file mode 100644 index 0000000..1839598 --- /dev/null +++ b/data/0.png diff --git a/data/1.png b/data/1.png Binary files differnew file mode 100644 index 0000000..981bdeb --- /dev/null +++ b/data/1.png diff --git a/data/2.png b/data/2.png Binary files differnew file mode 100644 index 0000000..4c6223f --- /dev/null +++ b/data/2.png diff --git a/data/2021-12-03 18-22-11.png b/data/2021-12-03 18-22-11.png Binary files differnew file mode 100644 index 0000000..048d954 --- /dev/null +++ b/data/2021-12-03 18-22-11.png diff --git a/data/3.png b/data/3.png Binary files differnew file mode 100644 index 0000000..f64a796 --- /dev/null +++ b/data/3.png diff --git a/data/4.png b/data/4.png Binary files differnew file mode 100644 index 0000000..2fb37fa --- /dev/null +++ b/data/4.png diff --git a/data/5.png b/data/5.png Binary files differnew file mode 100644 index 0000000..f1f853c --- /dev/null +++ b/data/5.png diff --git a/data/6.png b/data/6.png Binary files differnew file mode 100644 index 0000000..a43b8f5 --- /dev/null +++ b/data/6.png diff --git a/data/7.png b/data/7.png Binary files differnew file mode 100644 index 0000000..49965a6 --- /dev/null +++ b/data/7.png diff --git a/data/8.png b/data/8.png Binary files differnew file mode 100644 index 0000000..62cb7e2 --- /dev/null +++ b/data/8.png diff --git a/data/9.png b/data/9.png Binary files differnew file mode 100644 index 0000000..9f43ec3 --- /dev/null +++ b/data/9.png diff --git a/data/test.png b/data/test.png Binary files differnew file mode 100644 index 0000000..0492aaa --- /dev/null +++ b/data/test.png diff --git a/index.html b/index.html new file mode 100644 index 0000000..b7c0153 --- /dev/null +++ b/index.html @@ -0,0 +1,91 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <title>queue go brrr</title> + <style> + #video { + border: 1px solid black; + box-shadow: 2px 2px 3px black; + height:240px; + } + + #photo { + border: 1px solid black; + box-shadow: 2px 2px 3px black; + height:240px; + } + + #canvas { + display:none; + } + + .camera { + width: 340px; + display:inline-block; + } + + .output { + width: 340px; + display:inline-block; + } + + #startbutton { + display:block; + position:relative; + margin-left:auto; + margin-right:auto; + bottom:32px; + background-color: rgba(0, 150, 0, 0.5); + border: 1px solid rgba(255, 255, 255, 0.7); + box-shadow: 0 0 1px 2px rgba(0, 0, 0, 0.2); + font-size: 14px; + font-family: "Lucida Grande", "Arial", sans-serif; + color: rgba(255, 255, 255, 1.0); + } + + main { + font-size: 16px; + font-family: "Lucida Grande", "Arial", sans-serif; + width: 760px; + } + </style> +</head> +<body> +<main> + <button id="bootbutton">Start!</button> + <div class="camera"> + <video id="video">Video stream not available.</video> + <button id="startbutton">Take photo</button> + </div> + <canvas id="canvas"> + </canvas> + <div class="output"> + <img id="photo" alt="The screen capture will appear in this box."> + </div> +</main> +<!-- Note the usage of `type=module` here as this is an ES6 module --> +<script type="module"> + // Use ES module import syntax to import functionality from the module + // that we have compiled. + // + // Note that the `default` import is an initialization function which + // will "boot" the module and make it ready to use. Currently browsers + // don't support natively imported WebAssembly as an ES module, but + // eventually the manual initialization won't be required! + import init, { boot } from './pkg/queue_go_brrr.js'; + + async function run() { + await init(); + + const bootButton = document.getElementById('bootbutton'); + bootButton.addEventListener('click', () => { + bootButton.remove(); + boot(); + }); + } + + run(); +</script> +</body> +</html>
\ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..8f28199 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,158 @@ +use image::{DynamicImage, ImageFormat}; +use std::rc::Rc; +use std::sync::atomic::Ordering::Relaxed; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + +use js_sys::{ArrayBuffer, Uint8Array}; +use wasm_bindgen::prelude::*; +use wasm_bindgen::JsCast; +use wasm_bindgen_futures::*; +use web_sys::*; + +mod ocr; + +// When the `wee_alloc` feature is enabled, use `wee_alloc` as the global +// allocator. +#[cfg(feature = "wee_alloc")] +#[global_allocator] +static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; + +async fn blob_to_image(blob: Blob) -> DynamicImage { + let array_buffer_promise = blob.array_buffer(); + let array_buffer: ArrayBuffer = JsFuture::from(array_buffer_promise) + .await + .unwrap() + .dyn_into() + .unwrap(); + let image_array = Uint8Array::new(&array_buffer); + let image_data = image_array.to_vec(); + image::load_from_memory_with_format(&image_data, ImageFormat::Png).unwrap() +} + +fn image_to_data_uri(image: &DynamicImage) -> String { + let mut result = String::from("data:image/png;base64,"); + let mut enc = base64::write::EncoderStringWriter::from(&mut result, base64::STANDARD); + image.write_to(&mut enc, ImageFormat::Png).unwrap(); + let _ = enc.into_inner(); + result +} + +fn get_queue_size(image: DynamicImage) -> DynamicImage { + let useful_region = image.crop_imm(911, 514, 70, 20); + + useful_region +} + +#[cfg(test)] +#[test] +fn test_queue_size() { + let image = image::open("data/2021-12-03 18-22-11.png").unwrap(); + let queue_size = get_queue_size(image); + queue_size.save("data/test.png").unwrap(); + let result = ocr::ocr(queue_size); + assert_eq!(result, 4_480); +} + +async fn do_boot() { + let width = Rc::new(AtomicU32::new(0)); + let height = Rc::new(AtomicU32::new(0)); + + let streaming = AtomicBool::new(false); + + let window = window().unwrap(); + let document = window.document().unwrap(); + + let video: HtmlVideoElement = document + .get_element_by_id("video") + .unwrap() + .dyn_into() + .unwrap(); + let canvas: HtmlCanvasElement = document + .get_element_by_id("canvas") + .unwrap() + .dyn_into() + .unwrap(); + let photo = document.get_element_by_id("photo").unwrap(); + let start_button = document.get_element_by_id("startbutton").unwrap(); + + let navigator = window.navigator(); + let media_devices = navigator.media_devices().unwrap(); + let constraints = { + let mut constraints = DisplayMediaStreamConstraints::new(); + constraints.audio(&JsValue::FALSE); + constraints.video(&JsValue::TRUE); + constraints + }; + let capture_stream_promise = media_devices + .get_display_media_with_constraints(&constraints) + .unwrap(); + let capture_stream = JsFuture::from(capture_stream_promise).await.unwrap(); + let capture_stream: MediaStream = capture_stream.dyn_into().unwrap(); + video.set_src_object(Some(&capture_stream)); + let video_listener = { + let width = width.clone(); + let height = height.clone(); + let video = video.clone(); + let also_video = video.clone(); + let canvas = canvas.clone(); + gloo::events::EventListener::new(&video, "canplay", move |_event| { + let _ = also_video.play().unwrap(); + if !streaming.load(Relaxed) { + width.store(also_video.video_width(), Relaxed); + height.store(also_video.video_height(), Relaxed); + canvas.set_attribute("width", &format!("{}", width.load(Relaxed))); + canvas.set_attribute("height", &format!("{}", height.load(Relaxed))); + streaming.store(true, Relaxed); + } + }) + }; + video_listener.forget(); + + let start_button_event_listener = { + let photo = photo.clone(); + gloo::events::EventListener::new(&start_button, "click", move |_event| { + let context = canvas.get_context("2d").unwrap().unwrap(); + let context: CanvasRenderingContext2d = context.dyn_into().unwrap(); + canvas.set_width(width.load(Ordering::Relaxed)); + canvas.set_height(height.load(Ordering::Relaxed)); + context + .draw_image_with_html_video_element_and_dw_and_dh( + &video, + 0.0, + 0.0, + width.load(Relaxed).into(), + height.load(Relaxed).into(), + ) + .unwrap(); + + canvas + .to_blob( + Closure::once_into_js(|data: Blob| { + console::log_1(&JsValue::from("hewwo?")); + // lifetime bullshit here + let window = web_sys::window().unwrap(); + let document = window.document().unwrap(); + let photo = document.get_element_by_id("photo").unwrap(); + spawn_local(async move { + let image = blob_to_image(data).await; + let useful_region = get_queue_size(image); + let data_uri = image_to_data_uri(&useful_region); + photo.set_attribute("src", &data_uri).unwrap(); + console::log_1(&JsValue::from("uhhhh hi")); + }) + }) + .dyn_ref() + .unwrap(), + ) + .unwrap(); + }) + }; + start_button_event_listener.forget(); +} + +#[wasm_bindgen] +pub fn boot() { + console_error_panic_hook::set_once(); + + spawn_local(do_boot()); +} diff --git a/src/ocr.rs b/src/ocr.rs new file mode 100644 index 0000000..bc7e535 --- /dev/null +++ b/src/ocr.rs @@ -0,0 +1,50 @@ +use image::{DynamicImage, GrayImage, ImageFormat}; +use imageproc::template_matching::MatchTemplateMethod; + +const REFERENCE_PNGS: [&[u8]; 10] = [ + include_bytes!("../data/0.png"), + include_bytes!("../data/1.png"), + include_bytes!("../data/2.png"), + include_bytes!("../data/3.png"), + include_bytes!("../data/4.png"), + include_bytes!("../data/5.png"), + include_bytes!("../data/6.png"), + include_bytes!("../data/7.png"), + include_bytes!("../data/8.png"), + include_bytes!("../data/9.png"), +]; + +fn x_matches(image: &GrayImage, template: &GrayImage) -> Vec<u32> { + let match_values = imageproc::template_matching::match_template( + image, + template, + MatchTemplateMethod::CrossCorrelationNormalized, + ); + match_values + .enumerate_pixels() + .filter(|(_x, _y, pix)| pix.0[0] > 0.9) + .map(|(x, _y, _pix)| x) + .collect() +} + +pub fn ocr(image: DynamicImage) -> u32 { + let grayscale_image = image::imageops::grayscale(&image); + let mut digit_x_positions: Vec<(u8, u32)> = (0..10) + .flat_map(|i| { + let png_i = + image::load_from_memory_with_format(REFERENCE_PNGS[i as usize], ImageFormat::Png) + .unwrap(); + let grey_png_i = image::imageops::grayscale(&png_i); + x_matches(&grayscale_image, &grey_png_i) + .into_iter() + .map(move |x| (i, x)) + }) + .collect(); + digit_x_positions.sort_by_key(|&(_i, x)| x); + let digits: String = digit_x_positions + .into_iter() + .map(|(i, _x)| format!("{}", i)) + .collect(); + dbg!(&digits); + digits.parse().unwrap() +} diff --git a/tests/web.rs b/tests/web.rs new file mode 100644 index 0000000..de5c1da --- /dev/null +++ b/tests/web.rs @@ -0,0 +1,13 @@ +//! Test suite for the Web and headless browsers. + +#![cfg(target_arch = "wasm32")] + +extern crate wasm_bindgen_test; +use wasm_bindgen_test::*; + +wasm_bindgen_test_configure!(run_in_browser); + +#[wasm_bindgen_test] +fn pass() { + assert_eq!(1 + 1, 2); +} |