|
Size: 1286
Comment: Faster parallel copy
|
← Revision 11 as of 2019-04-28 00:16:11 ⇥
Size: 1376
Comment: Check MP3s using parallel, recursively
|
| Deletions are marked like this. | Additions are marked like this. |
| Line 10: | Line 10: |
| # Likewise, but recursively find . -iname '*mp3' -print0 | parallel -0 -X mp3check -Sqe |
1 # Parallel copy
2 # Because we are copying the current directory, make sure $dst is correct
3 parallel --bar -j4 cp -avn -- "{}" "$dst" ::: *
4 # Old idea, much slower
5 find . -maxdepth 1 -mindepth 1 -print0 | parallel -q0 -j4 echo cp -avn "{}" "$dst"
6
7 # Check MP3s in parallel, many arguments per invocation
8 parallel -X mp3check -Sqe ::: *mp3
9 # Likewise, but recursively
10 find . -iname '*mp3' -print0 | parallel -0 -X mp3check -Sqe
11
12 # Run commands in a file parallel
13 parallel < commands.txt
14
15 # Using a file (one per line) of arguments
16 parallel 'some-command {}' ::: list-of-arguments.txt
17
18 # Parallel equivalent of
19 # `find $DIRECTORY -type f -exec sha1sum '{}' \; > $DIRECTORY.sha1`
20 find $DIRECTORY -type f -print0 | parallel -q0 -k sha1sum > $DIRECTORY.sha1
21
22 # Quickly get hash of large file. Reads in 1G chunks, multiple CPUs, and outputs 1 hash.
23 parallel --block=1G --pipepart -a $LARGE_FILE --progress --recend '' -k sha1sum | sha1sum
-X: Multiple arguments per job, spread evenly across jobs, with context replace to replace "{}".
-m: Multiple arguments per job, spread evenly across jobs. Preferably use -X.
--xargs: Multiple arguments per job emulating xargs behavior; as many arguments per job as possible
-n: Limits number of arguments per job, use w/ -X, --xargs, or -m
