From d8f568b1bcaa60c5345dce61b6cfc46cd178f408 Mon Sep 17 00:00:00 2001 From: Milosz Linkiewicz Date: Mon, 7 Oct 2024 07:12:14 +0000 Subject: [PATCH 1/3] Reflecting Dockerfile-netbootxyz changes Reflecting Dockerfile-netbootxyz changes. Split the `start.sh` entrypoint file: - init.sh - prepares environment and directories on first run - start.sh - entrypoint script that runs supervisord and init.sh Made basic set of parameters (ports) configurable. Now the `default` file need to be templated before first run. Minor fixes due to supervisor failing. Added proxy settings from environment pass to WebApplication Signed-off-by: Milosz Linkiewicz --- root/defaults/default | 2 +- root/etc/supervisor.conf | 6 ++-- root/init.sh | 65 ++++++++++++++++++++++++++++++++++ root/start.sh | 76 ++-------------------------------------- 4 files changed, 71 insertions(+), 78 deletions(-) create mode 100755 root/init.sh diff --git a/root/defaults/default b/root/defaults/default index a5a0a08..a31c772 100644 --- a/root/defaults/default +++ b/root/defaults/default @@ -1,5 +1,5 @@ server { - listen 80; + listen ${NGINX_PORT}; location / { root /assets; autoindex on; diff --git a/root/etc/supervisor.conf b/root/etc/supervisor.conf index 986420e..1168254 100644 --- a/root/etc/supervisor.conf +++ b/root/etc/supervisor.conf @@ -15,14 +15,14 @@ daemon=off priority = 2 [program:webapp] -environment=NODE_ENV="production",PORT=3000 +environment=NODE_ENV="production",PORT="%(ENV_WEB_APP_PORT)s",HTTPS_PROXY="%(ENV_HTTPS_PROXY)s",HTTP_PROXY="%(ENV_HTTP_PROXY)s",NO_PROXY="%(ENV_NO_PROXY)s" command=/usr/bin/node app.js user=nbxyz directory=/app priority = 3 -[program:in.tftpd] -command=/usr/sbin/in.tftpd -Lvvv --user nbxyz --secure %(ENV_TFTPD_OPTS)s /config/menus +[program:dnsmasq] +command=/usr/sbin/dnsmasq --port=0 --keep-in-foreground --enable-tftp --user=nbxyz --tftp-secure --tftp-root=/config/menus %(ENV_TFTPD_OPTS)s stdout_logfile=/config/tftpd.log redirect_stderr=true priority = 4 diff --git a/root/init.sh b/root/init.sh new file mode 100755 index 0000000..d778f6d --- /dev/null +++ b/root/init.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +set -exo pipefail +SCRIPT_DIR="$(readlink -f "$(dirname -- "${BASH_SOURCE[0]}")")" + +# Leave empty for standard flow or any other value for config recreate. +RECREATE_CONFIGURATION="${RECREATE_CONFIGURATION:-''}" + +# make config, logs, nginx etc. dirs +mkdir -p /var/lib/nginx/tmp/client_body /var/tmp/nginx \ + /config/menus/remote /config/menus/local \ + /config/nginx/site-confs /config/log/nginx /assets /run + +# Check for file exisitance, and depending on environment replace/create/nothing +[[ -n "${RECREATE_CONFIGURATION}" ]] && \ + rm -f /config/nginx/nginx.conf /config/nginx/site-confs/* + +[[ ! -f /config/nginx/nginx.conf ]] && \ + cp /defaults/nginx.conf /config/nginx/nginx.conf + +[[ ! -f /config/nginx/site-confs/default ]] && \ + envsubst < /defaults/default > /config/nginx/site-confs/default + +# Ownership +chown -R nbxyz:nbxyz /assets /var/lib/nginx /var/log/nginx + +# download menus if not found +if [[ ! -f /config/menus/remote/menu.ipxe ]]; then + echo "[netbootxyz-init] Downloading netboot.xyz at ${MENU_VERSION}" + echo "[netbootxyz-init] Downloading Menus at ${MENU_VERSION}" + curl -o /config/endpoints.yml -sL \ + "https://raw.githubusercontent.com/netbootxyz/netboot.xyz/${MENU_VERSION}/endpoints.yml" + curl -o /tmp/menus.tar.gz -sL \ + "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/menus.tar.gz" + tar xf /tmp/menus.tar.gz -C /config/menus/remote + + # boot files + echo "[netbootxyz-init] Downloading boot files at ${MENU_VERSION}" + curl -o /config/menus/remote/netboot.xyz.kpxe \ + -sL "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz.kpxe" + curl -o /config/menus/remote/netboot.xyz-undionly.kpxe \ + -sL "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz-undionly.kpxe" + curl -o /config/menus/remote/netboot.xyz.efi \ + -sL "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz.efi" + curl -o /config/menus/remote/netboot.xyz-snp.efi \ + -sL "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz-snp.efi" + curl -o /config/menus/remote/netboot.xyz-snponly.efi \ + -sL "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz-snponly.efi" + curl -o /config/menus/remote/netboot.xyz-arm64.efi \ + -sL "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz-arm64.efi" + curl -o /config/menus/remote/netboot.xyz-arm64-snp.efi \ + -sL "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz-arm64-snp.efi" + curl -o /config/menus/remote/netboot.xyz-arm64-snponly.efi \ + -sL "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz-arm64-snponly.efi" + + # layer and cleanup + echo "[netbootxyz-init] layer and cleanup " + + echo -n "${MENU_VERSION}" > /config/menuversion.txt + cp -r /config/menus/remote/* /config/menus + rm -f /tmp/menus.tar.gz +fi + +# Ownership +chown -R nbxyz:nbxyz /config diff --git a/root/start.sh b/root/start.sh index 2aab5dc..0c2d9ff 100755 --- a/root/start.sh +++ b/root/start.sh @@ -1,79 +1,7 @@ #!/bin/bash -# make our folders -mkdir -p \ - /assets \ - /config/nginx/site-confs \ - /config/log/nginx \ - /run \ - /var/lib/nginx/tmp/client_body \ - /var/tmp/nginx - -# copy config files -[[ ! -f /config/nginx/nginx.conf ]] && \ - cp /defaults/nginx.conf /config/nginx/nginx.conf -[[ ! -f /config/nginx/site-confs/default ]] && \ - cp /defaults/default /config/nginx/site-confs/default - -# Ownership -chown -R nbxyz:nbxyz /assets -chown -R nbxyz:nbxyz /var/lib/nginx -chown -R nbxyz:nbxyz /var/log/nginx - -# create local logs dir -mkdir -p \ - /config/menus/remote \ - /config/menus/local - -# download menus if not found -if [[ ! -f /config/menus/remote/menu.ipxe ]]; then - if [[ -z ${MENU_VERSION+x} ]]; then \ - MENU_VERSION=$(curl -sL "https://api.github.com/repos/netbootxyz/netboot.xyz/releases/latest" | jq -r '.tag_name') - fi - echo "[netbootxyz-init] Downloading netboot.xyz at ${MENU_VERSION}" - # menu files - curl -o \ - /config/endpoints.yml -sL \ - "https://raw.githubusercontent.com/netbootxyz/netboot.xyz/${MENU_VERSION}/endpoints.yml" - curl -o \ - /tmp/menus.tar.gz -sL \ - "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/menus.tar.gz" - tar xf \ - /tmp/menus.tar.gz -C \ - /config/menus/remote - # boot files - curl -o \ - /config/menus/remote/netboot.xyz.kpxe -sL \ - "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz.kpxe" - curl -o \ - /config/menus/remote/netboot.xyz-undionly.kpxe -sL \ - "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz-undionly.kpxe" - curl -o \ - /config/menus/remote/netboot.xyz.efi -sL \ - "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz.efi" - curl -o \ - /config/menus/remote/netboot.xyz-snp.efi -sL \ - "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz-snp.efi" - curl -o \ - /config/menus/remote/netboot.xyz-snponly.efi -sL \ - "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz-snponly.efi" - curl -o \ - /config/menus/remote/netboot.xyz-arm64.efi -sL \ - "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz-arm64.efi" - curl -o \ - /config/menus/remote/netboot.xyz-arm64-snp.efi -sL \ - "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz-arm64-snp.efi" - curl -o \ - /config/menus/remote/netboot.xyz-arm64-snponly.efi -sL \ - "https://github.com/netbootxyz/netboot.xyz/releases/download/${MENU_VERSION}/netboot.xyz-arm64-snponly.efi" - # layer and cleanup - echo -n ${MENU_VERSION} > /config/menuversion.txt - cp -r /config/menus/remote/* /config/menus - rm -f /tmp/menus.tar.gz -fi - -# Ownership -chown -R nbxyz:nbxyz /config +# Perform the initial configuration +/init.sh echo " _ _ _ " echo " _ __ ___| |_| |__ ___ ___ | |_ __ ___ _ ____ " From 88f97d503c4477f06112f932992ccdeaf207cdb2 Mon Sep 17 00:00:00 2001 From: Milosz Linkiewicz Date: Mon, 7 Oct 2024 07:16:35 +0000 Subject: [PATCH 2/3] HTTP and HTTPS proxy agents added HTTP and HTTPS proxy agents added. Now every link connection before cration is checked against environment variables: - http_proxy/HTTP_PROXY, - https_proxy/HTTPS_PROXY, - no_proxy/NO_PROXY, It is determined if connection should be routed throug proxy server. This includes: - DownloaderHelper. - Socket fetch. - Devgetbrowser method. Fix to Readdirp handling after migration to v4.0.0. Signed-off-by: Milosz Linkiewicz --- app.js | 60 ++++++++++++++++++++++++++++++++++++++++------------ package.json | 9 +++++--- 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/app.js b/app.js index 2ef0f4d..d9dc902 100644 --- a/app.js +++ b/app.js @@ -4,6 +4,10 @@ var baseurl = process.env.SUBFOLDER || '/'; var app = require('express')(); var { DownloaderHelper } = require('node-downloader-helper'); +const { HttpProxyAgent } = require('http-proxy-agent'); +const { HttpsProxyAgent } = require('https-proxy-agent'); +const { URL } = require('url'); +const getProxyForUrl = require('proxy-from-env').getProxyForUrl; var exec = require('child_process').exec; var express = require('express'); var fs = require('fs'); @@ -11,7 +15,7 @@ var http = require('http').Server(app); var io = require('socket.io')(http, {path: baseurl + 'socket.io'}); var isBinaryFile = require("isbinaryfile").isBinaryFile; var path = require('path'); -var readdirp = require('readdirp'); +let {readdirp} = require('readdirp'); var fetch = require('node-fetch'); var si = require('systeminformation'); const util = require('util'); @@ -34,6 +38,26 @@ function disablesigs(){ } } +function getProxyAgentFromUrl(request_url, request_options=false) { + const proxy_url = getProxyForUrl(request_url); + if(!proxy_url) { + return false; + } + + const protocol = new URL(request_url).protocol; + let agent; + + if (protocol === 'https:') { + agent = new HttpsProxyAgent(proxy_url); + return request_options ? { httpsRequestOptions: { agent } } : agent; + } else if (protocol === 'http:') { + agent = new HttpProxyAgent(proxy_url); + return request_options ? { httpRequestOptions: { agent } } : agent; + } + + return false; +} + ////// PATHS ////// //// Main //// baserouter.get("/", function (req, res) { @@ -60,9 +84,11 @@ io.on('connection', function(socket){ var tftpcmd = '/usr/sbin/dnsmasq --version | head -n1'; var nginxcmd = '/usr/sbin/nginx -v'; var dashinfo = {}; + const fetch_uri = 'https://api.github.com/repos/netbootxyz/netboot.xyz/releases/latest'; + var fetch_opts = {headers:{'user-agent':'node.js'},agent:getProxyAgentFromUrl(fetch_uri,false)}; dashinfo['webversion'] = version; dashinfo['menuversion'] = fs.readFileSync('/config/menuversion.txt', 'utf8'); - fetch('https://api.github.com/repos/netbootxyz/netboot.xyz/releases/latest', {headers: {'user-agent': 'node.js'}}) + fetch(fetch_uri, fetch_opts) .then(response => { if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); @@ -83,10 +109,10 @@ io.on('connection', function(socket){ dashinfo['nginxversion'] = stderr; io.sockets.in(socket.id).emit('renderdash',dashinfo); }); - }); + }); }); }); - }); + }); }) .catch(error => { console.log('There was a problem with the fetch operation: ' + error.message); @@ -141,7 +167,7 @@ io.on('connection', function(socket){ io.sockets.in(socket.id).emit('renderconfig',remote_files,local_files); }); }); - // When a create file is + // When a create file is socket.on('createipxe', function(filename){ fs.writeFileSync('/config/menus/local/' + filename, '#!ipxe'); layermenu(function(response){ @@ -155,7 +181,14 @@ io.on('connection', function(socket){ var remotemenuversion = fs.readFileSync('/config/menuversion.txt', 'utf8'); var endpointsfile = fs.readFileSync('/config/endpoints.yml'); var endpoints = yaml.load(endpointsfile); - var localfiles = await readdirp.promise('/assets/.'); + // Wrap readdirp in a promise + var localfiles = await new Promise((resolve, reject) => { + const entries = []; + readdirp('/assets/.') + .on('data', (entry) => entries.push(entry)) + .on('end', () => resolve(entries)) + .on('error', (error) => reject(error)); + }); var assets = []; if (localfiles.length != 0){ for (var i in localfiles){ @@ -185,8 +218,8 @@ io.on('connection', function(socket){ }); // When Dev Browser is requested reach out to github for versions socket.on('devgetbrowser', async function(){ - var api_url = 'https://api.github.com/repos/netbootxyz/netboot.xyz/'; - var options = {headers: {'user-agent': 'node.js'}}; + const api_url = 'https://api.github.com/repos/netbootxyz/netboot.xyz/'; + var options = {headers:{'user-agent':'node.js'},agent:getProxyAgentFromUrl(api_url,false)}; var releasesResponse = await fetch(api_url + 'releases', options); if (!releasesResponse.ok) { throw new Error(`HTTP error! status: ${releasesResponse.status}`); @@ -251,7 +284,7 @@ async function upgrademenu(version, callback){ } for (var i in rom_files){ var file = rom_files[i]; - var url = download_endpoint + file; + var url = download_endpoint + file; downloads.push({'url':url,'path':remote_folder}); } // static config for endpoints @@ -295,10 +328,11 @@ async function downloader(downloads){ var value = downloads[i]; var url = value.url; var path = value.path; - var dloptions = {override:true,retry:{maxRetries:2,delay:5000}}; + var agent = getProxyAgentFromUrl(url, true); + var dloptions = Object.assign({}, {override:true,retry:{maxRetries:2,delay:5000}}, agent); var dl = new DownloaderHelper(url, path, dloptions); - dl.on('end', function(){ + dl.on('end', function(){ console.log('Downloaded ' + url + ' to ' + path); }); @@ -321,11 +355,11 @@ async function downloader(downloads){ if ( ! url.includes('s3.amazonaws.com')){ // Part 2 if exists repeat - var response = await fetch(url + '.part2', {method: 'HEAD'}); + var response = await fetch(url + '.part2',{method:'HEAD',agent:getProxyAgentFromUrl(url,false)}); var urltest = response.headers.get('server'); if (urltest == 'AmazonS3' || urltest == 'Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0') { var dl2 = new DownloaderHelper(url + '.part2', path, dloptions); - dl2.on('end', function(){ + dl2.on('end', function(){ console.log('Downloaded ' + url + '.part2' + ' to ' + path); }); dl2.on('progress', function(stats){ diff --git a/package.json b/package.json index a4d9fa9..5dc931f 100644 --- a/package.json +++ b/package.json @@ -15,14 +15,17 @@ "homepage": "https://netboot.xyz", "dependencies": { "ejs": "3.1.10", - "express": "4.19.2", + "express": "4.21.0", "http": "0.0.0", + "http-proxy-agent": "7.0.2", + "https-proxy-agent": "7.0.5", "isbinaryfile": "5.0.2", "js-yaml": "4.1.0", "node-downloader-helper": "2.1.9", - "readdirp": "3.6.0", + "proxy-from-env": "1.1.0", + "readdirp": "4.0.1", "node-fetch": "2.7.0", - "socket.io": "4.7.5", + "socket.io": "4.8.0", "systeminformation": "5.23.3" } } From c90823481e0230edf7ff0e4a1941b015f7633f9d Mon Sep 17 00:00:00 2001 From: Milosz Linkiewicz Date: Mon, 7 Oct 2024 07:19:15 +0000 Subject: [PATCH 3/3] Dockerfile code improvements and labeles added. Dockerfile code improvements and labeles added. The improvements include using bash as default shell for bettter error detection while performing install and start of container. Signed-off-by: Milosz Linkiewicz --- Dockerfile | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0b6c98b..328dd7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,21 @@ FROM alpine:3.18 # set version label -ARG BUILD_DATE -ARG VERSION -ARG WEBAPP_VERSION -LABEL build_version="netboot.xyz version: ${VERSION} Build-date: ${BUILD_DATE}" -LABEL maintainer="antonym" +ARG BUILD_DATE="10/7/2024" +ARG VERSION="0.7.3" +ARG WEBAPP_VERSION="0.7.3" + +LABEL org.opencontainers.image.authors="antony@mes.ser.li" +LABEL org.opencontainers.image.url="https://github.com/netbootxyz/webapp" +LABEL org.opencontainers.image.title="NetBoot.xyz WebApp" +LABEL org.opencontainers.image.description="NetBoot.xyz WebApp: A NodeJS helper application for managing local deployments of netboot.xyz" +LABEL org.opencontainers.image.documentation="https://netboot.xyz/docs/docker" +LABEL org.opencontainers.image.version="${WEBAPP_VERSION}" +LABEL org.opencontainers.image.vendor="https://NetBoot.xyz" +LABEL org.opencontainers.image.licenses="Apache-2.0 license" + +RUN apk add --no-cache bash +SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] RUN \ apk upgrade --no-cache && \ @@ -26,15 +36,13 @@ RUN \ supervisor \ syslog-ng \ tar \ - tftp-hpa - -RUN \ + tftp-hpa && \ groupmod -g 1000 users && \ useradd -u 911 -U -d /config -s /bin/false nbxyz && \ usermod -G users nbxyz && \ mkdir /app \ /config \ - /defaults + /defaults COPY . /app @@ -43,10 +51,12 @@ RUN \ ENV TFTPD_OPTS='' -EXPOSE 3000 -EXPOSE 8080 +EXPOSE 69/UDP 80/TCP 3000/TCP 8080/TCP +VOLUME ["/assets", "/config"] COPY root/ / +CMD ["/start.sh"] +SHELL ["/bin/bash", "-c"] # default command -CMD ["sh","/start.sh"] +ENTRYPOINT ["/start.sh"]