[Semio] Scratch Update/Fixes

.dockerignore

@@ -13,5 +13,7 @@ dependencies/emsdk/.emscripten.old
 dependencies/cpython/.git
 dependencies/cpython/builddir
 dependencies/libwallaby/.git
+dependencies/kipr-scratch/libwallaby-build
+dependencies/kipr-scratch/kipr-scratch
 
 **/.github

.github/workflows/ci.yml

@@ -14,6 +14,12 @@ jobs:
       uses: actions/checkout@v2
       with:
         path: simulator
+        submodules: recursive
+    - name: Install Simulator system dependencies
+      run: sudo apt-get update && sudo apt-get install -y wget git cmake build-essential swig zlib1g-dev doxygen default-jre python2.7
+    - name: Install Simulator dependencies
+      run: yarn run build-deps
+      working-directory: simulator
     - name: Install Simulator modules
       # Specifying an alternative cache folder to work around a race condition issue in yarn,
       # which seems to occur b/c of how ivygate is referenced from GitHub.

.gitignore

@@ -2,6 +2,7 @@
 node_modules
 yarn-error.log
 dist
+.yarncache
 
 # Prerequisites
 *.d

.gitmodules

@@ -7,4 +7,6 @@
 [submodule "dependencies/emsdk"]
 	path = dependencies/emsdk
 	url = https://github.com/emscripten-core/emsdk
-
+[submodule "dependencies/kipr-scratch"]
+	path = dependencies/kipr-scratch
+	url = https://github.com/kipr/kipr-scratch

Dockerfile

@@ -4,7 +4,7 @@ RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 ENV TZ=America/Los_Angeles
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
-RUN apt-get update && apt-get install -y wget git cmake build-essential python3.8 swig zlib1g-dev doxygen
+RUN apt-get update && apt-get install -y wget git cmake build-essential python3.8 swig zlib1g-dev doxygen default-jre python2.7
 
 RUN wget https://deb.nodesource.com/setup_18.x && chmod +x ./setup_18.x && ./setup_18.x
 RUN apt-get install -y nodejs
@@ -13,11 +13,12 @@ RUN npm install -g yarn
 
 ADD . /app
 
+EXPOSE 3000
 WORKDIR /app/
-RUN python3.8 dependencies/build.py
+RUN yarn run build-deps
+# RUN python3.8 dependencies/build.py
 
-WORKDIR /app
-EXPOSE 3000
+# WORKDIR /app
 # WORKDIR /app/simulator
 # RUN yarn install --cache-folder ./.yarncache && yarn build; true
 RUN yarn install --cache-folder ./.yarncache

README.md

@@ -16,6 +16,7 @@ Simulates a botball/JBC style demobot with a built in IDE.
 - [CMake](https://cmake.org/)
 - [SWIG 4+](https://swig.org/)
 - [Python 3.7 or newer](https://www.python.org/)
+- [Also python 2.7]
 
 ### Debian/Ubuntu
 ```bash
@@ -59,6 +60,7 @@ Or, if you've already cloned the repository without `--recurse-submodules`, you
 
 ```bash
 git submodule update --init
+cd dependencies/kipr-scratch/libwallaby && git submodule update --init
 ```
 
 ### Optional: Pull large files
@@ -71,11 +73,52 @@ git lfs pull
 
 ```bash
 # Python 3.7+ is required for the build process
-python3 dependencies/build.py
-```
+yarn run build-deps
+# The build-deps command executes:
+"python3 dependencies/build.py && yarn add file:dependencies/kipr-scratch/kipr-scratch --cache-folder ./.yarncache",
 
+```
 Tip: if you are experiencing issues with this step, you may try deleting the repository and follow the steps listed above again.
 
+### Notes on building dependencies
+```python3 dependencies/build.py```
+
+1. Run some setup checks
+2. Build libwallaby (probably only need to do this once)
+3. Delete ''unnecessary blocks' from scratch-blocks/blocks-vertical (renames them with a .old extension)
+    - 'event.js',
+    - 'extensions.js',
+    - 'default_toolbox.js',
+    - 'looks.js',
+    - 'motion.js',
+    - 'sensing.js',
+    - 'sound.js'
+
+
+4. blockify libwallaby
+    - Input is libwallaby-build, output is in scratch-blocks/blocks_vertical
+    - Start by parsing the xml result of the libwallaby build
+      - Essentially getting all the functions from the included files and cdecl nodes
+      - Ends up with a list of modules and their functions
+    - For all functions, needs to generate a blockly block of the module_function with
+      with the correct parameters and return types.
+      - Only does it for analog, digital, wait_for, time, motor, and servo modules
+    - Patches the colors in core/colours.js for the new modules from info in module_hsl.json
+    - Writes a new default_toolbox.js - this is what we have access to pick from
+    - Modifies the category names in the vertical extensions file, removing 'sounds', 'motion', 
+      'looks', 'event', 'sensing', 'pen', but adding our modules
+    - Modifies the messages to include those from our modules and the program start
+    - Modifies the css flyout style
+    - Modifies the field variables to remove non initialization case
+
+
+5. Patch the scratch-blocks package.json to use python2
+
+6. Install the scratch blocks
+
+
+
+
 ## Install JavaScript Dependencies
 
 Navigate to the root directory of this repository, then run:

configs/webpack/common.js

@@ -36,7 +36,8 @@ try {
 module.exports = {
   entry: {
     app: './index.tsx',
-    login: './components/Login/index.tsx',
+    login: './components/Login/index.tsx', 
+    plugin: './lms/plugin/index.tsx',
     'editor.worker': 'monaco-editor/esm/vs/editor/editor.worker.js',
     'ts.worker': 'monaco-editor/esm/vs/language/typescript/ts.worker.js',
   },
@@ -63,10 +64,11 @@ module.exports = {
       path: false,
     },
     alias: {
+      state: resolve(__dirname, '../../src/state'),
       '@i18n': resolve(__dirname, '../../src/util/i18n'),
     },
     symlinks: false,
-    modules
+    modules //: [resolve(__dirname, '../../src'), 'node_modules']
   },
   context: resolve(__dirname, '../../src'),
   module: {
@@ -135,14 +137,15 @@ module.exports = {
     ],
   },
   plugins: [
-    new HtmlWebpackPlugin({ template: 'index.html.ejs', excludeChunks: ['login'] }),
+    new HtmlWebpackPlugin({ template: 'index.html.ejs', excludeChunks: ['login', 'plugin'] }),
     new HtmlWebpackPlugin({ template: 'components/Login/login.html.ejs', filename: 'login.html', chunks: ['login'] }),
+    new HtmlWebpackPlugin({ template: 'lms/plugin/plugin.html.ejs', filename: 'plugin.html', chunks: ['plugin'] }),
     new DefinePlugin({
       SIMULATOR_VERSION: JSON.stringify(require('../../package.json').version),
       SIMULATOR_GIT_HASH: JSON.stringify(commitHash),
       SIMULATOR_HAS_CPYTHON: JSON.stringify(dependencies.cpython !== undefined),
       SIMULATOR_LIBKIPR_C_DOCUMENTATION: JSON.stringify(libkiprCDocumentation),
-      SIMULATOR_I18N: JSON.stringify(i18n),
+      SIMULATOR_I18N: JSON.stringify(i18n)
     }),
     new NpmDtsPlugin({
       root: resolve(__dirname, '../../'),

dependencies/.gitignore

@@ -1,5 +1,5 @@
 libkipr_build_c
 libkipr_build_python
 libkipr_install_c
-ammo_build
+scratch-rt.js
 dependencies.json

dependencies/build.py

@@ -5,6 +5,26 @@
 import pathlib
 import json
 import multiprocessing
+import hashlib
+
+def sha1OfFile(filepath):
+  sha = hashlib.sha1()
+  with open(filepath, 'rb') as f:
+    while True:
+      block = f.read(2**10) # Magic number: one-megabyte blocks.
+      if not block: break
+      sha.update(block)
+    return sha.hexdigest()
+
+def hash_dir(dir_path):
+  hashes = []
+  for path, dirs, files in os.walk(dir_path):
+    for file in sorted(files):
+      hashes.append(sha1OfFile(os.path.join(path, file)))
+    for dir in sorted(dirs):
+      hashes.append(hash_dir(os.path.join(path, dir)))
+    break
+  return str(hash(''.join(hashes)))
 
 def is_tool(name):
     """Check whether `name` is on PATH and marked as executable."""
@@ -68,6 +88,7 @@ def is_tool(name):
 }
 
 libkipr_dir = working_dir / 'libwallaby'
+libkipr_hash = hash_dir(libkipr_dir)
 
 # libkipr (C)
 
@@ -86,6 +107,7 @@ def is_tool(name):
     '-Dwith_python_binding=OFF',
     '-Dwith_documentation=ON',
     '-Dwith_tests=OFF',
+    '-Dwith_graphics=OFF',
     f'-DCMAKE_INSTALL_PREFIX={libkipr_install_c_dir}',
     libkipr_dir
   ],
@@ -200,12 +222,46 @@ def is_tool(name):
 print('Generating JSON documentation...')
 libkipr_c_documentation_json = f'{libkipr_build_c_dir}/documentation/json.json'
 subprocess.run(
-  [ 'python3', 'generate_doxygen_json.py', f'{libkipr_build_c_dir}/documentation/xml', libkipr_c_documentation_json ],
+  [ python, 'generate_doxygen_json.py', f'{libkipr_build_c_dir}/documentation/xml', libkipr_c_documentation_json ],
+  # [ 'python3', 'generate_doxygen_json.py', f'{libkipr_build_c_dir}/documentation/xml', libkipr_c_documentation_json ],
   cwd = working_dir,
   check = True
 )
 
+print('Building kipr-scratch...')
+kipr_scratch_path = working_dir / 'kipr-scratch'
+subprocess.run(
+  [ python, kipr_scratch_path / 'build.py' ],
+  cwd = kipr_scratch_path,
+  check = True
+)
 
+print('Packaging kipr-scratch...')
+subprocess.run(
+  [ python, kipr_scratch_path / 'package.py' ],
+  cwd = kipr_scratch_path,
+  check = True
+)
+
+print('Generating scratch runtime...')
+scratch_runtime_path = working_dir / 'scratch-rt'
+# emcc -s WASM=0 -s INVOKE_RUN=0 -s ASYNCIFY -s EXIT_RUNTIME=1 -s "EXPORTED_FUNCTIONS=['_main', '_simMainWrapper']" -I${config.server.dependencies.libkipr_c}/include -Wl,--whole-archive -L${config.server.dependencies.libkipr_c}/lib -lkipr -o ${path}.js ${path}
+subprocess.run([
+    'emcc',
+    '-sWASM=0',
+    '-sINVOKE_RUN=0',
+    '-sEXIT_RUNTIME=0',
+    '-sERROR_ON_UNDEFINED_SYMBOLS=0',
+    '-sLINKABLE=1',
+    '-sEXPORT_ALL=1',
+    f'-L{libkipr_install_c_dir}/lib',
+    '-Wl,--whole-archive', '-lkipr', '-Wl,--no-whole-archive',
+    f'-o', f'{scratch_runtime_path}.js',
+    f'{scratch_runtime_path}.c'
+  ],
+  env = env,
+  check = True
+)
 
 print('Outputting results...')
 output = json.dumps({
@@ -216,10 +272,13 @@ def is_tool(name):
     'EMSDK': f'{emsdk_dir}',
     'EM_CONFIG': f'{emsdk_dot_emscripten}'
   },
+  'libkipr_hash': libkipr_hash,
   'libkipr_c': f'{libkipr_install_c_dir}',
   'libkipr_python': f'{libkipr_build_python_dir}',
   'cpython': f'{cpython_emscripten_build_dir}',
+  'cpython_hash': hash_dir(cpython_dir),
   "libkipr_c_documentation": libkipr_c_documentation_json,
+  'scratch_rt': f'{scratch_runtime_path}.js',
 })
 
 with open(working_dir / 'dependencies.json', 'w') as f:

dependencies/scratch-rt.c

@@ -0,0 +1 @@
+int main() { return 0; }

express.js

@@ -11,6 +11,7 @@ const sourceDir = 'dist';
 const { get: getConfig } = require('./config');
 const { WebhookClient } = require('discord.js');
 const proxy = require('express-http-proxy');
+const path = require('path');
 
 
 let config;
@@ -21,6 +22,16 @@ try {
   throw e;
 }
 
+// set up rate limiter: maximum of 100 requests per 15 minute
+var RateLimit = require('express-rate-limit');
+var limiter = RateLimit({
+  windowMs: 15 * 60 * 1000, // 15 minutes
+  max: 100, // max 100 requests per windowMs
+});
+
+// apply rate limiter to all requests
+app.use(limiter);
+
 app.use((req, res, next) => {
   res.header("Access-Control-Allow-Origin", "*");
   res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
@@ -204,6 +215,22 @@ app.use('/static', express.static(`${__dirname}/static`, {
   maxAge: config.caching.staticMaxAge,
 }));
 
+
+if (config.server.dependencies.scratch_rt) {
+  console.log('Scratch Runtime is enabled.');
+  app.use('/scratch/rt.js', express.static(`${config.server.dependencies.scratch_rt}`, {
+    maxAge: config.caching.staticMaxAge,
+  }));
+}
+
+app.use('/scratch', express.static(path.resolve(__dirname, 'node_modules', 'kipr-scratch'), {
+  maxAge: config.caching.staticMaxAge,
+}));
+
+app.use('/media', express.static(path.resolve(__dirname, 'node_modules', 'kipr-scratch', 'media'), {
+  maxAge: config.caching.staticMaxAge,
+}));
+
 // Expose cpython artifacts
 if (config.server.dependencies.cpython) {
   console.log('CPython artifacts are enabled.');
@@ -233,6 +260,11 @@ app.get('/login', (req, res) => {
   res.sendFile(`${__dirname}/${sourceDir}/login.html`);
 });
 
+
+app.get('/lms/plugin', (req, res) => {
+  res.sendFile(`${__dirname}/${sourceDir}/plugin.html`);
+});
+
 app.use('*', (req, res) => {
   setCrossOriginIsolationHeaders(res);
   res.sendFile(`${__dirname}/${sourceDir}/index.html`);