From 04273c2422ed5cb8a9fa6b492637a426fb86a40f Mon Sep 17 00:00:00 2001 From: Jakub Fojt Date: Thu, 9 Mar 2023 11:15:53 +0100 Subject: [PATCH] Break tools out into modules --- .bashrc.kuba | 99 ++------------------------------- .modules.kuba/.slurm-aliases.sh | 33 +++++++++++ .modules.kuba/.tools.sh | 96 ++++++++++++++++++++++++++++++++ .modules.kuba/slurm-aliases.lua | 24 ++------ .modules.kuba/tools.lua | 11 ++++ 5 files changed, 149 insertions(+), 114 deletions(-) create mode 100644 .modules.kuba/.slurm-aliases.sh create mode 100644 .modules.kuba/.tools.sh create mode 100644 .modules.kuba/tools.lua diff --git a/.bashrc.kuba b/.bashrc.kuba index 7ed0de9..841945e 100755 --- a/.bashrc.kuba +++ b/.bashrc.kuba @@ -35,105 +35,10 @@ complete -o bashdefault -o default -o nospace -F __git_wrap__git_main dotfiles # Define a OS X-like open command open() { command xdg-open "$@" > /dev/null 2>&1 & } -# Inspect .npz files -npz_py_str="$(cat << EOM -from sys import argv -import numpy as np -try: - import matplotlib.pyplot as plt - print('Imported matplotlib.pyplot as plt') -except ImportError: - pass - -class Wrap: - def __init__(self, fname): - n = np.load(fname, allow_pickle=True) - files = n.files - globs = globals() - print('Contains files', files) - for f in files: - setattr(self, f, n[f]) - globs[f] = n[f] - -argv = argv[argv.index('--')+1:] -archives = [Wrap(arg) for arg in argv[::-1]] -archive = archives[0] -print('Global namespace populated with "archive", "archives" and all files') -EOM -)" - -npz_preview() { - if [ $# -lt 1 ]; then - echo "Please specify file to open as argument" - return - fi - - python -ic "$npz_py_str" -- "$@" -} - -npz_previewi() { - if [ $# -lt 1 ]; then - echo "Please specify file to open as argument" - return - fi - - ipython -i -c "$npz_py_str" -- "$@" -} - -rsync_wrap() { - cmd="rsync -a -R -r --copy-links --exclude .git --exclude __pycache__ --progress" - if [ $# == 0 ]; then - echo "Usage rsync_wrap [...] - -Useful for synchronizing repositories - -Calls $cmd [...]" - return - fi - $cmd $@ -} - -rsync_wrap_huge() { - cmd="rsync -a -R -r --copy-links --size-only --exclude .git --exclude __pycache__ --progress" - if [ $# == 0 ]; then - echo "Usage rsync_wrap_huge [...] - -Useful for synchronizing repositories. Does not do checksum - -Calls $cmd [...]" - return - fi - $cmd $@ -} - -watch_modify() { - if [ $# -lt 2 ]; then - echo "Usage watch_modify FILE CMD ARGS... - - Run CMD ARGS... when file FILE is updated - " - return - fi - - file="$1" - base="$(basename "$file")" - dir="$(dirname "$file")" - inotifywait -r -m -e modify "$dir" | while read file_path file_event file_name; do - [ "$base" == "$file_name" ] && echo ${@:2} && ${@:2} - done -} - # Source git prompt source_existing ~/scripts/git-prompt.sh source_existing ~/.bash_prompt.kuba -if command -v xclip &> /dev/null; then - alias selc="xclip -sel primary -i" - alias selv="xclip -sel primary -o" - alias clipc="xclip -sel clip -i" - alias clipv="xclip -sel clip -o" -fi - # Activate python virtualenv source_existing ~/.python-venv.kuba/bin/activate @@ -153,5 +58,9 @@ if [ -z "$DONT_LOAD_LMOD" ]; then module use "$HOME/.modules.kuba/" module use "$HOME/git/auto-venv-setup/lmod-modules/" + module load tools + else + # lmod does not exist + source $HOME/.modules.kuba/.tools.sh fi fi diff --git a/.modules.kuba/.slurm-aliases.sh b/.modules.kuba/.slurm-aliases.sh new file mode 100644 index 0000000..19d303c --- /dev/null +++ b/.modules.kuba/.slurm-aliases.sh @@ -0,0 +1,33 @@ +qnode() { + [ $# -lt 1 ] && echo "Usage qnode JOBID" && return 0 + squeue --noheader --jobs $1 -o %R +} + +qssh() { + [ $# -lt 1 ] && echo "Usage qssh JOBID" && return 0 + ssh $(qnode $1) +} + +taillog() { [ $# -lt 1 ] && echo "Usage taillog LOGFILE [JOBIDs ..]" && return 0 + if [ $# -lt 2 ] ; then + tail "$1" + else + tail "$1" "$1.$2"{.e,.o} + fi +} + +catlog() { [ $# -lt 1 ] && echo "Usage catlog LOGFILE [JOBIDs ..]" && return 0 + if [ $# -lt 2 ] ; then + cat "$1" | less + else + cat "$1" "$1.$2"{.e,.o} | less + fi +} + +watchlog() { [ $# -lt 2 ] && echo "Usage watchlog INTERVAL LOGFILE [JOBIDs ..]" && return 0; + if [ $# -lt 3 ] ; then + watch -n $1 tail "$2" + else + watch -n $1 tail "$2" "$2.$3"{.e,.o} + fi +} diff --git a/.modules.kuba/.tools.sh b/.modules.kuba/.tools.sh new file mode 100644 index 0000000..074730c --- /dev/null +++ b/.modules.kuba/.tools.sh @@ -0,0 +1,96 @@ +# Inspect .npz files +npz_py_str="$(cat << EOM +from sys import argv +import numpy as np +try: + import matplotlib.pyplot as plt + print('Imported matplotlib.pyplot as plt') +except ImportError: + pass + +class Wrap: + def __init__(self, fname): + n = np.load(fname, allow_pickle=True) + files = n.files + globs = globals() + print('Contains files', files) + for f in files: + setattr(self, f, n[f]) + globs[f] = n[f] + +argv = argv[argv.index('--')+1:] +archives = [Wrap(arg) for arg in argv] +archive = archives[-1] +print('Global namespace populated with "archive", "archives" and all files') +for i, fname in enumerate(argv): + print(f' {i:<3.0f}: {fname}') +EOM +)" + +npz_preview() { + if [ $# -lt 1 ]; then + echo "Please specify file to open as argument" + return + fi + + python -ic "$npz_py_str" -- "$@" +} + +npz_previewi() { + if [ $# -lt 1 ]; then + echo "Please specify file to open as argument" + return + fi + + ipython -i -c "$npz_py_str" -- "$@" +} + +rsync_wrap() { + cmd="rsync -a -R -r --copy-links --exclude .git --exclude __pycache__ --progress" + if [ $# == 0 ]; then + echo "Usage rsync_wrap [...] + +Useful for synchronizing repositories + +Calls $cmd [...]" + return + fi + $cmd $@ +} + +rsync_wrap_huge() { + cmd="rsync -a -R -r --copy-links --size-only --exclude .git --exclude __pycache__ --progress" + if [ $# == 0 ]; then + echo "Usage rsync_wrap_huge [...] + +Useful for synchronizing repositories. Does not do checksum + +Calls $cmd [...]" + return + fi + $cmd $@ +} + +if command -v xclip &> /dev/null; then + alias selc="xclip -sel primary -i" + alias selv="xclip -sel primary -o" + alias clipc="xclip -sel clip -i" + alias clipv="xclip -sel clip -o" +fi + +watch_modify() { + if [ $# -lt 2 ]; then + echo "Usage watch_modify FILE CMD ARGS... + + Run CMD ARGS... when file FILE is updated + " + return + fi + + file="$1" + base="$(basename "$file")" + dir="$(dirname "$file")" + inotifywait -r -m -e modify "$dir" | while read file_path file_event file_name; do + [ "$base" == "$file_name" ] && echo ${@:2} && ${@:2} + done +} diff --git a/.modules.kuba/slurm-aliases.lua b/.modules.kuba/slurm-aliases.lua index 47cdccc..7200d69 100644 --- a/.modules.kuba/slurm-aliases.lua +++ b/.modules.kuba/slurm-aliases.lua @@ -9,22 +9,8 @@ set_alias("qa", 'squeue -o "%8A %10u %.6Q %24j %2t %3r %16S %.10M %.10L %.4D %16 set_alias("qid", 'squeue -h -u $USER -o "%A"') set_alias("scancelall", 'squeue -h -u $USER -o "%A" | xargs scancel') -execute{cmd='qnode() { [ $# -lt 1 ] && echo "Usage qnode JOBID" && return 0; '.. - 'squeue --noheader --jobs $1 -o %R; '.. - '}', modeA={"load"}} -execute{cmd='qssh() { [ $# -lt 1 ] && echo "Usage qssh JOBID" && return 0; '.. - 'ssh $(qnode $1); '.. - '}', modeA={"load"}} -execute{cmd='taillog() { [ $# -lt 1 ] && echo "Usage taillog LOGFILE [JOBIDs ..]" && return 0; '.. - 'if [ $# -lt 2 ] ; then tail "$1" ; ' .. - 'else tail "$1" "$1.$2"{.e,.o} ;' .. - 'fi }', modeA={"load"}} -execute{cmd='catlog() { [ $# -lt 1 ] && echo "Usage catlog LOGFILE [JOBIDs ..]" && return 0; '.. - 'if [ $# -lt 2 ] ; then cat "$1" | less ; ' .. - 'else cat "$1" "$1.$2"{.e,.o} | less ;' .. - 'fi }', modeA={"load"}} -execute{cmd='watchlog() { [ $# -lt 2 ] && echo "Usage watchlog INTERVAL LOGFILE [JOBIDs ..]" && return 0; '.. - 'if [ $# -lt 3 ] ; then watch -n $1 tail "$2"; ' .. - 'else watch -n $1 tail "$2" "$2.$3"{.e,.o}; ' .. - 'fi }', modeA={"load"}} -execute{cmd='unset -f qnode qssh watchlog catlog taillog', modeA={"unload"}} +local path = splitFileName(myFileName()); +local loadscript = pathJoin(path, '.' .. myModuleName() .. '.sh'); + +execute{cmd='source ' .. loadscript, modeA={"load"}} +execute{cmd='unset -f qnode qssh watchlog catlog taillog;', modeA={"unload"}} diff --git a/.modules.kuba/tools.lua b/.modules.kuba/tools.lua new file mode 100644 index 0000000..31f4079 --- /dev/null +++ b/.modules.kuba/tools.lua @@ -0,0 +1,11 @@ +whatis([==[ +Loads useful tools +]==]) + +local path = splitFileName(myFileName()); +local loadscript = pathJoin(path, '.' .. myModuleName() .. '.sh'); + +execute{cmd='source ' .. loadscript, modeA={"load"}} +execute{cmd='unset -f npz_preview npz_previewi ' .. + 'rsync_wrap rsync_wrap_huge selc selv cliipc clipv watch_modify' .. + ';', modeA={"unload"}}