From 53f28e45b0a83019f9c988b5d23990a26585a700 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Sat, 7 Dec 2024 20:55:06 +0100 Subject: [PATCH 01/19] Strip off variables (?both). Allow multiple input files. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index a0a15b3b..4e54ef4f 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -9,10 +9,10 @@ usage() { - echo "Usage: check_links.sh [-d] [-a] mdfile" - echo "The script will search the mdfile for pictures from hedgedoc and report the missing ones" + echo "Usage: check_links.sh [-d] [-a] mdfile [mdfile [...]]" + echo "The script will search the mdfile(s) for pictures from hedgedoc and report the missing ones" echo "The option -d will also download the missing ones and suggest you git add them." - echo "The option -a will not limit the links to be downloaded to images." + echo "The option -a will not limit the links to images but all linked files." exit 1 } @@ -41,15 +41,18 @@ if test "$1" = "-d"; then DOWNLOAD=1; shift; fi if test "$1" = "-a"; then ALL=1; shift; fi if test -z "$1" -o "$1" = "-h"; then usage; fi if test ! -r "$1"; then echo "ERROR: File \"$1\" not readable" 1>&2; exit 2; fi + +ADDS="" +errs=0 +while test -n "$1"; do INPATH=${1%/*} INFILE=${1##*/} +pushd . if test "$INPATH" != "$1"; then cd $INPATH; INPATH="${INPATH}/"; else INPATH=""; fi -ADDS="" -errs=0 while read line; do # FIXME: Handle multiple links in one line - LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #"]*\).*$@\1@')" + LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #?"]*\).*$@\1@')" LINK="${LINK%\'}" if ispic $LINK; then if exist $LINK; then @@ -66,6 +69,10 @@ while read line; do fi fi done < <(tr ' ' '\n' < "$INFILE"|grep 'https://input.scs.community'|sed 's/!\[..*\]//g') +shift +popd +done + if test "$DOWNLOAD" -a -n "$ADDS"; then echo -e "Consider\ngit add $ADDS" fi From 666309220a4f7fcb3ee30026f02bedb09dca34ce Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Sun, 8 Dec 2024 16:50:44 +0100 Subject: [PATCH 02/19] Use download API /download to grab .md files. Name them .md then also for better rendering. Replace links to hedgedoc by link to the local .md file. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index 4e54ef4f..692faaca 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -44,12 +44,15 @@ if test ! -r "$1"; then echo "ERROR: File \"$1\" not readable" 1>&2; exit 2; fi ADDS="" errs=0 + +SEDCHANGES="" while test -n "$1"; do INPATH=${1%/*} INFILE=${1##*/} -pushd . +pushd . >/dev/null 2>&1 if test "$INPATH" != "$1"; then cd $INPATH; INPATH="${INPATH}/"; else INPATH=""; fi +CHANGES="" while read line; do # FIXME: Handle multiple links in one line LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #?"]*\).*$@\1@')" @@ -60,8 +63,21 @@ while read line; do else echo "$LINK missing" if test "$DOWNLOAD" = "1"; then - curl -LO "$LINK" - if test $? = "0"; then echo "Downloaded $LINK successfully."; ADDS="$ADDS ${INPATH}${LINK##*/}" + if ispic $LINK; then + curl -LO "$LINK" + ERR=$? + TGTFILE=${LINK##*/} + else + curl -LO "$LINK"/download + ERR=$? + TGTFILE=${LINK##*/}.md + mv download $TGTFILE + fi + if test $ERR = "0"; then + echo "Downloaded $LINK successfully." + ADDS="$ADDS ${INPATH}${LINK##*/}" + LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #"]*\).*$@\1@')" + CHANGES="$CHANGES -e 's~$LINK~./$TGTFILE~g'" else echo "ERROR downloading $LINK" 1>&2; let errs+=1; fi else let errs+=1 @@ -69,12 +85,14 @@ while read line; do fi fi done < <(tr ' ' '\n' < "$INFILE"|grep 'https://input.scs.community'|sed 's/!\[..*\]//g') +if test -n "$CHANGES"; then SEDCHANGES="$SEDCHANGES sed -i $CHANGES $1;"; fi shift -popd +popd >/dev/null 2>&1 done if test "$DOWNLOAD" -a -n "$ADDS"; then echo -e "Consider\ngit add $ADDS" + echo "$SEDCHANGES" fi if test $errs -gt 0; then echo "$errs missing files" 1>&2 From 662425b0dc95d89ab97d0d8f67b826841eaa7985 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Mon, 9 Dec 2024 16:57:18 +0100 Subject: [PATCH 03/19] Use new filename, with an attached .md if appropriate. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index 692faaca..d9c46edb 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -75,9 +75,10 @@ while read line; do fi if test $ERR = "0"; then echo "Downloaded $LINK successfully." - ADDS="$ADDS ${INPATH}${LINK##*/}" + ADDS="$ADDS ${INPATH}${TGTFILE}" + # We strip off variables here, but leave anchors in LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #"]*\).*$@\1@')" - CHANGES="$CHANGES -e 's~$LINK~./$TGTFILE~g'" + CHANGES="$CHANGES -e 's~${LINK}~./${TGTFILE}~g'" else echo "ERROR downloading $LINK" 1>&2; let errs+=1; fi else let errs+=1 From 0b60751b10b784d9aded90671cba0fe250f94b43 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Mon, 9 Dec 2024 22:46:02 +0100 Subject: [PATCH 04/19] Move check for $ALL outside of ispic() check. Otherwise we can't reuse the function ... Signed-off-by: Kurt Garloff --- tools/check_links.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index d9c46edb..6a4af85d 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -18,7 +18,6 @@ usage() ispic() { - if test "$ALL" = "1"; then return 0; fi ext=${1##*.} ext="$(echo $ext | tr A-Z a-z)" case $ext in @@ -57,7 +56,7 @@ while read line; do # FIXME: Handle multiple links in one line LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #?"]*\).*$@\1@')" LINK="${LINK%\'}" - if ispic $LINK; then + if test "$ALL" = "1" || ispic $LINK; then if exist $LINK; then echo "$LINK present already" else From a3fa11ff3a7dc2df8883f207f030b6cff793f80b Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Mon, 9 Dec 2024 23:05:18 +0100 Subject: [PATCH 05/19] Also change links to already downloaded refs. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index 6a4af85d..3e102e50 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -30,8 +30,7 @@ ispic() exist() { - fn=${1##*/} - test -r $fn + test -r "$fn" } # main @@ -56,36 +55,40 @@ while read line; do # FIXME: Handle multiple links in one line LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #?"]*\).*$@\1@')" LINK="${LINK%\'}" + TGTFILE="${LINK##*/}" if test "$ALL" = "1" || ispic $LINK; then - if exist $LINK; then - echo "$LINK present already" + ERR=0 + if exist "$TGTFILE"; then + echo "$TGTFILE present already" + elif exist "$TGTFILE.md"; then + TGTFILE="$TGTFILE.md" + echo "$TGTFILE present already" else echo "$LINK missing" if test "$DOWNLOAD" = "1"; then if ispic $LINK; then curl -LO "$LINK" ERR=$? - TGTFILE=${LINK##*/} else curl -LO "$LINK"/download ERR=$? - TGTFILE=${LINK##*/}.md - mv download $TGTFILE + TGTFILE="${LINK##*/}.md" + mv download "$TGTFILE" fi - if test $ERR = "0"; then + if test $ERR = 0; then echo "Downloaded $LINK successfully." ADDS="$ADDS ${INPATH}${TGTFILE}" - # We strip off variables here, but leave anchors in - LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #"]*\).*$@\1@')" - CHANGES="$CHANGES -e 's~${LINK}~./${TGTFILE}~g'" - else echo "ERROR downloading $LINK" 1>&2; let errs+=1; fi - else - let errs+=1 + fi fi - fi + fi + if test $ERR = 0; then + # We strip off variables here, but leave anchors in + LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #"]*\).*$@\1@')" + CHANGES="$CHANGES -e 's~${LINK}~./${TGTFILE}~g'" + else echo "ERROR downloading $LINK" 1>&2; let errs+=1; fi fi done < <(tr ' ' '\n' < "$INFILE"|grep 'https://input.scs.community'|sed 's/!\[..*\]//g') -if test -n "$CHANGES"; then SEDCHANGES="$SEDCHANGES sed -i $CHANGES $1;"; fi +if test -n "$CHANGES"; then SEDCHANGES="${SEDCHANGES}sed -i $CHANGES $1; "; fi shift popd >/dev/null 2>&1 done From 7266e5ba19c8ba20d4f75af2617612c1f9926520 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Mon, 9 Dec 2024 23:17:18 +0100 Subject: [PATCH 06/19] Avoid downloading empty files. Quoting. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index 3e102e50..81200d04 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -18,7 +18,7 @@ usage() ispic() { - ext=${1##*.} + ext="${1##*.}" ext="$(echo $ext | tr A-Z a-z)" case $ext in png|jpg|jpeg|heic|heif|svg|avif) @@ -50,14 +50,17 @@ INFILE=${1##*/} pushd . >/dev/null 2>&1 if test "$INPATH" != "$1"; then cd $INPATH; INPATH="${INPATH}/"; else INPATH=""; fi +echo "*** $INFILE ***" CHANGES="" while read line; do # FIXME: Handle multiple links in one line LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #?"]*\).*$@\1@')" LINK="${LINK%\'}" TGTFILE="${LINK##*/}" - if test "$ALL" = "1" || ispic $LINK; then + if test -z "$TGTFILE"; then continue; fi + if test "$ALL" = "1" || ispic "$LINK"; then ERR=0 + echo " Consider replacing $LINK with $TGTFILE[.md] ..." if exist "$TGTFILE"; then echo "$TGTFILE present already" elif exist "$TGTFILE.md"; then @@ -66,11 +69,11 @@ while read line; do else echo "$LINK missing" if test "$DOWNLOAD" = "1"; then - if ispic $LINK; then - curl -LO "$LINK" + if ispic "$LINK"; then + curl -sLO "$LINK" ERR=$? else - curl -LO "$LINK"/download + curl -sLO "$LINK"/download ERR=$? TGTFILE="${LINK##*/}.md" mv download "$TGTFILE" From 252b77b5fc0a13eed1b4edba4f17ebea1cc859f0 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Mon, 9 Dec 2024 23:38:23 +0100 Subject: [PATCH 07/19] Add link anchors if needed. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index 81200d04..04c890c1 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -56,15 +56,15 @@ while read line; do # FIXME: Handle multiple links in one line LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #?"]*\).*$@\1@')" LINK="${LINK%\'}" - TGTFILE="${LINK##*/}" + TGTFILE="${LINK##*/}" if test -z "$TGTFILE"; then continue; fi if test "$ALL" = "1" || ispic "$LINK"; then - ERR=0 + ERR=0 echo " Consider replacing $LINK with $TGTFILE[.md] ..." if exist "$TGTFILE"; then echo "$TGTFILE present already" - elif exist "$TGTFILE.md"; then - TGTFILE="$TGTFILE.md" + elif exist "$TGTFILE.md"; then + TGTFILE="$TGTFILE.md" echo "$TGTFILE present already" else echo "$LINK missing" @@ -83,12 +83,18 @@ while read line; do ADDS="$ADDS ${INPATH}${TGTFILE}" fi fi - fi - if test $ERR = 0; then - # We strip off variables here, but leave anchors in - LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #"]*\).*$@\1@')" - CHANGES="$CHANGES -e 's~${LINK}~./${TGTFILE}~g'" - else echo "ERROR downloading $LINK" 1>&2; let errs+=1; fi + fi + if test $ERR = 0; then + # We strip off variables here, but leave anchors in + LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #"]*\).*$@\1@')" + LINK2="$(echo $line | sed 's@^.*(\(https://input.scs.community/[^) #"]*\)).*$@\1@')" + LINK3="$(echo $line | sed 's@^.*<\(https://input.scs.community/[^) #"]*\)>.*$@\1@')" + if test "$LINK" = "$LINK2" -o "$LINK" = "$LINK3"; then + CHANGES="$CHANGES -e 's~${LINK}~./${TGTFILE}~g'" + else + CHANGES="$CHANGES -e 's~${LINK}~[${TGTFILE%#*}](./${TGTFILE})~g'" + fi + else echo "ERROR downloading $LINK" 1>&2; let errs+=1; fi fi done < <(tr ' ' '\n' < "$INFILE"|grep 'https://input.scs.community'|sed 's/!\[..*\]//g') if test -n "$CHANGES"; then SEDCHANGES="${SEDCHANGES}sed -i $CHANGES $1; "; fi From 7f6bdf3e1962982f1be7134f53779d76af5dd21d Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 10 Dec 2024 00:11:23 +0100 Subject: [PATCH 08/19] Better detection of enclosing <> and (). Signed-off-by: Kurt Garloff --- tools/check_links.sh | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index 04c890c1..75d29ba3 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -54,18 +54,23 @@ echo "*** $INFILE ***" CHANGES="" while read line; do # FIXME: Handle multiple links in one line - LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #?"]*\).*$@\1@')" - LINK="${LINK%\'}" - TGTFILE="${LINK##*/}" + LINK1="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) "]*\).*$@\1@')" + LINK="${LINK1%\'}" + LINKANCHOR="${LINK##*#}" + if test "$LINK" = "$LINKANCHOR"; then LINKANCHOR=""; fi + LINKNOANCHOR="${LINK%#*}" + LINKNOVAR="${LINK%\?*}" + LINKNONO="${LINKNOANCHOR%\?*}" + TGTFILE="${LINKNONO##*/}" if test -z "$TGTFILE"; then continue; fi if test "$ALL" = "1" || ispic "$LINK"; then ERR=0 echo " Consider replacing $LINK with $TGTFILE[.md] ..." if exist "$TGTFILE"; then - echo "$TGTFILE present already" + echo " $TGTFILE present already" elif exist "$TGTFILE.md"; then TGTFILE="$TGTFILE.md" - echo "$TGTFILE present already" + echo " $TGTFILE present already" else echo "$LINK missing" if test "$DOWNLOAD" = "1"; then @@ -73,26 +78,26 @@ while read line; do curl -sLO "$LINK" ERR=$? else - curl -sLO "$LINK"/download + curl -sLO "$LINKNONO"/download ERR=$? - TGTFILE="${LINK##*/}.md" + TGTFILE="${TGTFILE}.md" mv download "$TGTFILE" fi if test $ERR = 0; then - echo "Downloaded $LINK successfully." + echo " Downloaded $LINK successfully." ADDS="$ADDS ${INPATH}${TGTFILE}" fi fi fi if test $ERR = 0; then + if test -n "$LINKANCHOR"; then TGTFILE="$TGTFILE#$LINKANCHOR"; fi # We strip off variables here, but leave anchors in - LINK="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) #"]*\).*$@\1@')" - LINK2="$(echo $line | sed 's@^.*(\(https://input.scs.community/[^) #"]*\)).*$@\1@')" - LINK3="$(echo $line | sed 's@^.*<\(https://input.scs.community/[^) #"]*\)>.*$@\1@')" - if test "$LINK" = "$LINK2" -o "$LINK" = "$LINK3"; then - CHANGES="$CHANGES -e 's~${LINK}~./${TGTFILE}~g'" + if echo "$line" | grep "(${LINK})" >/dev/null; then + CHANGES="$CHANGES -e 's~${LINK}~${TGTFILE}~g'" + elif echo "$link" | grep "<$LINK>" >/dev/null; then + CHANGES="$CHANGES -e 's~<${LINK}[^>]*>~[${TGTFILE%#*}](${TGTFILE}~g'" else - CHANGES="$CHANGES -e 's~${LINK}~[${TGTFILE%#*}](./${TGTFILE})~g'" + CHANGES="$CHANGES -e 's~${LINK}~[${TGTFILE%#*}](${TGTFILE})~g'" fi else echo "ERROR downloading $LINK" 1>&2; let errs+=1; fi fi From a9e97edcb6cf41888bcb21893e5e0dbad67aef1b Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 10 Dec 2024 00:20:24 +0100 Subject: [PATCH 09/19] lso git add files changed by sed. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index 75d29ba3..fa88bef5 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -41,6 +41,7 @@ if test -z "$1" -o "$1" = "-h"; then usage; fi if test ! -r "$1"; then echo "ERROR: File \"$1\" not readable" 1>&2; exit 2; fi ADDS="" +CHGD="" errs=0 SEDCHANGES="" @@ -102,14 +103,17 @@ while read line; do else echo "ERROR downloading $LINK" 1>&2; let errs+=1; fi fi done < <(tr ' ' '\n' < "$INFILE"|grep 'https://input.scs.community'|sed 's/!\[..*\]//g') -if test -n "$CHANGES"; then SEDCHANGES="${SEDCHANGES}sed -i $CHANGES $1; "; fi +if test -n "$CHANGES"; then SEDCHANGES="${SEDCHANGES}sed -i $CHANGES $1; "; CHGD="$CHGD $1"; fi shift popd >/dev/null 2>&1 done if test "$DOWNLOAD" -a -n "$ADDS"; then - echo -e "Consider\ngit add $ADDS" - echo "$SEDCHANGES" + echo -e "Consider\ngit add$ADDS" + if test -n "$CHGD"; then + echo "$SEDCHANGES" + echo "git add$CHGD" + fi fi if test $errs -gt 0; then echo "$errs missing files" 1>&2 From 6a608618e2e794ac200d9e1cc1d6b47c2250dca3 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 10 Dec 2024 06:35:10 +0100 Subject: [PATCH 10/19] Fix test for existing files. Strip trailing ws. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index fa88bef5..edc2038d 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -30,7 +30,7 @@ ispic() exist() { - test -r "$fn" + test -r "$1" } # main @@ -69,8 +69,8 @@ while read line; do echo " Consider replacing $LINK with $TGTFILE[.md] ..." if exist "$TGTFILE"; then echo " $TGTFILE present already" - elif exist "$TGTFILE.md"; then - TGTFILE="$TGTFILE.md" + elif exist "${TGTFILE}.md"; then + TGTFILE="${TGTFILE}.md" echo " $TGTFILE present already" else echo "$LINK missing" @@ -82,10 +82,11 @@ while read line; do curl -sLO "$LINKNONO"/download ERR=$? TGTFILE="${TGTFILE}.md" + sed -i 's/\s*$//' download mv download "$TGTFILE" fi if test $ERR = 0; then - echo " Downloaded $LINK successfully." + echo " Downloaded $LINKNONO successfully." ADDS="$ADDS ${INPATH}${TGTFILE}" fi fi From 60b4606c21597b73c292dcb18c8c722f601b8255 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 10 Dec 2024 07:06:29 +0100 Subject: [PATCH 11/19] ALso output changed files if none got added. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index edc2038d..07e025fb 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -95,26 +95,31 @@ while read line; do if test -n "$LINKANCHOR"; then TGTFILE="$TGTFILE#$LINKANCHOR"; fi # We strip off variables here, but leave anchors in if echo "$line" | grep "(${LINK})" >/dev/null; then + #echo "() -> Plain replacement" CHANGES="$CHANGES -e 's~${LINK}~${TGTFILE}~g'" - elif echo "$link" | grep "<$LINK>" >/dev/null; then + elif echo "$line" | grep "<$LINK>" >/dev/null; then + #echo "<> -> Change to []()" CHANGES="$CHANGES -e 's~<${LINK}[^>]*>~[${TGTFILE%#*}](${TGTFILE}~g'" else + #echo "Plain -> Change to []()" CHANGES="$CHANGES -e 's~${LINK}~[${TGTFILE%#*}](${TGTFILE})~g'" fi - else echo "ERROR downloading $LINK" 1>&2; let errs+=1; fi + else + echo "ERROR downloading $LINK" 1>&2; let errs+=1 + fi fi done < <(tr ' ' '\n' < "$INFILE"|grep 'https://input.scs.community'|sed 's/!\[..*\]//g') -if test -n "$CHANGES"; then SEDCHANGES="${SEDCHANGES}sed -i $CHANGES $1; "; CHGD="$CHGD $1"; fi +if test -n "$CHANGES"; then SEDCHANGES="${SEDCHANGES}sed -i$CHANGES $1; "; CHGD="$CHGD $1"; fi shift popd >/dev/null 2>&1 done -if test "$DOWNLOAD" -a -n "$ADDS"; then +if test -n "$DOWNLOAD" -a -n "$ADDS"; then echo -e "Consider\ngit add$ADDS" - if test -n "$CHGD"; then - echo "$SEDCHANGES" - echo "git add$CHGD" - fi +fi +if test -n "$DOWNLOAD" -a -n "$CHGD"; then + echo "$SEDCHANGES" + echo "git add$CHGD" fi if test $errs -gt 0; then echo "$errs missing files" 1>&2 From d8227e1ac5a7c556ef8c258b8ddab85e0d932069 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 10 Dec 2024 08:19:18 +0100 Subject: [PATCH 12/19] Also replace self-references. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index 07e025fb..4a245f25 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -63,7 +63,13 @@ while read line; do LINKNOVAR="${LINK%\?*}" LINKNONO="${LINKNOANCHOR%\?*}" TGTFILE="${LINKNONO##*/}" - if test -z "$TGTFILE"; then continue; fi + if test -z "$TGTFILE"; then + if test -n "$LINKANCHOR" -a -n "#DOWNLOAD"; then + echo " Replace link to self with anchor $LINKANCHOR ..." + CHANGES="$CHANGES -e 's~${LINK}~#{LINKANCHOR}~g'" + fi + continue + fi if test "$ALL" = "1" || ispic "$LINK"; then ERR=0 echo " Consider replacing $LINK with $TGTFILE[.md] ..." From 2c76322819da8a6e1eb9c1ae6d04a226e145f16e Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 10 Dec 2024 08:32:50 +0100 Subject: [PATCH 13/19] Skip .css and .md files. Fix replacing lonely anchors. Also output sed command line by line. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index 4a245f25..f0dab77e 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -66,10 +66,12 @@ while read line; do if test -z "$TGTFILE"; then if test -n "$LINKANCHOR" -a -n "#DOWNLOAD"; then echo " Replace link to self with anchor $LINKANCHOR ..." - CHANGES="$CHANGES -e 's~${LINK}~#{LINKANCHOR}~g'" + CHANGES="$CHANGES -e 's~${LINK}~#${LINKANCHOR}~g'" fi continue fi + # Skip .css and .js + if test "${TGTFILE%.css}" = "$TGTFILE" -o "${TGTFILE%.js}" = "$TGTFILE"; then continue; fi if test "$ALL" = "1" || ispic "$LINK"; then ERR=0 echo " Consider replacing $LINK with $TGTFILE[.md] ..." @@ -115,7 +117,7 @@ while read line; do fi fi done < <(tr ' ' '\n' < "$INFILE"|grep 'https://input.scs.community'|sed 's/!\[..*\]//g') -if test -n "$CHANGES"; then SEDCHANGES="${SEDCHANGES}sed -i$CHANGES $1; "; CHGD="$CHGD $1"; fi +if test -n "$CHANGES"; then SEDCHANGES="${SEDCHANGES}sed -i$CHANGES \"$1\"\n"; CHGD="$CHGD \"$1\""; fi shift popd >/dev/null 2>&1 done @@ -124,7 +126,7 @@ if test -n "$DOWNLOAD" -a -n "$ADDS"; then echo -e "Consider\ngit add$ADDS" fi if test -n "$DOWNLOAD" -a -n "$CHGD"; then - echo "$SEDCHANGES" + echo -en "$SEDCHANGES" echo "git add$CHGD" fi if test $errs -gt 0; then From e6e32a075b4bf2e9095965c691936e8a765d26f7 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 10 Dec 2024 08:36:25 +0100 Subject: [PATCH 14/19] Fix logic in lookgin for *.js and *.cdd Signed-off-by: Kurt Garloff --- tools/check_links.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index f0dab77e..0ebd201c 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -71,7 +71,7 @@ while read line; do continue fi # Skip .css and .js - if test "${TGTFILE%.css}" = "$TGTFILE" -o "${TGTFILE%.js}" = "$TGTFILE"; then continue; fi + if test "${TGTFILE%.css}" != "$TGTFILE" -o "${TGTFILE%.js}" != "$TGTFILE"; then continue; fi if test "$ALL" = "1" || ispic "$LINK"; then ERR=0 echo " Consider replacing $LINK with $TGTFILE[.md] ..." From 73b0298f57ed78463a86e2f5db093672d4ba20cd Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 10 Dec 2024 08:45:33 +0100 Subject: [PATCH 15/19] Strip out ">" from URL. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index 0ebd201c..d6b9625a 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -55,7 +55,7 @@ echo "*** $INFILE ***" CHANGES="" while read line; do # FIXME: Handle multiple links in one line - LINK1="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) "]*\).*$@\1@')" + LINK1="$(echo $line | sed 's@^.*\(https://input.scs.community/[^) ">]*\).*$@\1@')" LINK="${LINK1%\'}" LINKANCHOR="${LINK##*#}" if test "$LINK" = "$LINKANCHOR"; then LINKANCHOR=""; fi @@ -70,8 +70,10 @@ while read line; do fi continue fi - # Skip .css and .js - if test "${TGTFILE%.css}" != "$TGTFILE" -o "${TGTFILE%.js}" != "$TGTFILE"; then continue; fi + # Skip .css and .js and .xml + if test "${TGTFILE%.css}" != "$TGTFILE" \ + -o "${TGTFILE%.js}" != "$TGTFILE" \ + -o "${TGTFILE%.xml}" != "$TGTFILE"; then continue; fi if test "$ALL" = "1" || ispic "$LINK"; then ERR=0 echo " Consider replacing $LINK with $TGTFILE[.md] ..." @@ -95,7 +97,7 @@ while read line; do fi if test $ERR = 0; then echo " Downloaded $LINKNONO successfully." - ADDS="$ADDS ${INPATH}${TGTFILE}" + ADDS="$ADDS \"${INPATH}${TGTFILE}\"" fi fi fi From 016825010e309fad4cced3e6ab3eec2bcc967484 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 10 Dec 2024 08:50:25 +0100 Subject: [PATCH 16/19] Fix missing closing ) when reaplcing <> linss. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index d6b9625a..09139996 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -109,7 +109,7 @@ while read line; do CHANGES="$CHANGES -e 's~${LINK}~${TGTFILE}~g'" elif echo "$line" | grep "<$LINK>" >/dev/null; then #echo "<> -> Change to []()" - CHANGES="$CHANGES -e 's~<${LINK}[^>]*>~[${TGTFILE%#*}](${TGTFILE}~g'" + CHANGES="$CHANGES -e 's~<${LINK}[^>]*>~[${TGTFILE%#*}](${TGTFILE})~g'" else #echo "Plain -> Change to []()" CHANGES="$CHANGES -e 's~${LINK}~[${TGTFILE%#*}](${TGTFILE})~g'" From b851e305a9ba832c62f8b1c99afb145307afb9d3 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 10 Dec 2024 08:55:42 +0100 Subject: [PATCH 17/19] Strip off extra .md. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index 09139996..f561c285 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -91,7 +91,7 @@ while read line; do else curl -sLO "$LINKNONO"/download ERR=$? - TGTFILE="${TGTFILE}.md" + TGTFILE="${TGTFILE%.md}.md" sed -i 's/\s*$//' download mv download "$TGTFILE" fi From bc2448bed435dc2970f6574e01a10fb4a30c986a Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 10 Dec 2024 10:56:45 +0100 Subject: [PATCH 18/19] We may not be allowed to download a file, detect this. We get an HTML from hedgedoc then, which we can detect and report. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index f561c285..10b6326e 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -41,6 +41,7 @@ if test -z "$1" -o "$1" = "-h"; then usage; fi if test ! -r "$1"; then echo "ERROR: File \"$1\" not readable" 1>&2; exit 2; fi ADDS="" +FAILED="" CHGD="" errs=0 @@ -91,9 +92,14 @@ while read line; do else curl -sLO "$LINKNONO"/download ERR=$? - TGTFILE="${TGTFILE%.md}.md" - sed -i 's/\s*$//' download - mv download "$TGTFILE" + if test $ERR=0 && grep '^$' download >/dev/null; then + ERR=1 + rm download + else + TGTFILE="${TGTFILE%.md}.md" + sed -i 's/\s*$//' download + mv download "$TGTFILE" + fi fi if test $ERR = 0; then echo " Downloaded $LINKNONO successfully." @@ -115,7 +121,9 @@ while read line; do CHANGES="$CHANGES -e 's~${LINK}~[${TGTFILE%#*}](${TGTFILE})~g'" fi else - echo "ERROR downloading $LINK" 1>&2; let errs+=1 + echo "ERROR downloading $LINK" 1>&2 + let errs+=1 + FAILED="$FAILED\"$LINK\" " fi fi done < <(tr ' ' '\n' < "$INFILE"|grep 'https://input.scs.community'|sed 's/!\[..*\]//g') @@ -133,5 +141,6 @@ if test -n "$DOWNLOAD" -a -n "$CHGD"; then fi if test $errs -gt 0; then echo "$errs missing files" 1>&2 + echo "FAILED: $FAILED" fi exit $errs From ddee8a663ee4a10340cc2651dd5be766a7a86c03 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 10 Dec 2024 15:11:02 +0100 Subject: [PATCH 19/19] Recognize lowercase HTML. Remove /p from slides for download. Signed-off-by: Kurt Garloff --- tools/check_links.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/check_links.sh b/tools/check_links.sh index 10b6326e..90de53f9 100755 --- a/tools/check_links.sh +++ b/tools/check_links.sh @@ -90,9 +90,11 @@ while read line; do curl -sLO "$LINK" ERR=$? else + # TODO: Handle hedgedoc /p links (/s does not need speciel treatment) + LINKNONO="${LINKNONO/https:\/\/input.scs.community\/p/https://input.scs.community}" curl -sLO "$LINKNONO"/download ERR=$? - if test $ERR=0 && grep '^$' download >/dev/null; then + if test $ERR=0 && grep -i '^$' download >/dev/null; then ERR=1 rm download else