这是我的问题。我是一名西班牙语翻译，我有一个很长的西班牙语-英语词汇表文件——50K 条目。此外，我有一个超过 1K 条目的停用词词汇表。我想从我计划翻译的文本中删除这些条目。因此，我构建了一个 sed 脚本，该脚本又从词汇表中构建了另外两个 sed 脚本，它们进行剥离，只留下未翻译的文本（因此我不必两次解决相同的问题）。这很好用，但问题是长文本需要很长时间，有时超过 15 分钟。这只是不可避免的，还是有更有效的方法来做到这一点？
这是主要脚本：
#!/bin/sh
before="$(date +%s)"
#wordstxt=$(wc -w < $1)
#mintime=$(expr "$wordstxt / 200" |bc -l)
#maxtime=$(expr "$wordstxt / 175" |bc -l)
#echo "Estimated time to process: between $mintime and $maxtime seconds."
sed '
s/\,/\n/g # strip all commas
s/\?/\n/g # strip question marks
s/\*/\n/g # strip asterisks
s/\!/\n/g # strip exclamation marks
s/:/\n/g # strip colons
s/\-/\n/g # strip hyphens
s/\./\n/g # strip periods
s/«/\n/g # strip left Euro-quotes
s/»/\n/g # strip right Euro-quotes
s/”/\n/g # strip slanted US quotes
s/\"/\n/g # strip left quotes
s/(/\n/g # strip left paren
s/)/\n/g # strip right paren
s/\[/\n/g # strip left bracket
s/\]/\n/g # strip right bracket
s/¿/\n/g # "¿"
s/—/\n/g # m-dash
s/\ –\ /\n/g # n-dash
s/…/\n/g # strip elipsis as a single character, not three periods
s/;/\n/g # strip semicolon
s/[0-9]/\n/g # strip out all numbers, replace with returns
' $1 > $1.z.tmp
#echo "Punctuation eliminated."
#cp ../../Spanish\ to\ English\ projects/glossary/stoplist.txt .
sed '
s/^\ //g # strip leading spaces
s/\ $// # strip trailing spaces
/^$/d # delete blank lines
s/\./\n/g # strip periods
s/\ /\\ /g # make spaces into literals
s/^/s\// # begins the substitution
s/$/\/\\n\/g/ # concludes the substitution
1 s/^/#!\ \/bin\/sed\ \-f\n\ns\/\[0\-9\]\/\/g\ns\/\\\ \\\ \/\\\ \/g\ns\/\\\.\\\ \/\\n\/g\n\n/
' stoplist.txt > stoplist.sed
chmod +x stoplist.sed
echo "Eliminating stopwords."
./stoplist.sed $1.z.tmp > $1.0.tmp
sed 's/\([A-Za-z\ ]*\t\).*/\1/' SpanishGlossary.utf8 > tempgloss.2.txt
#echo "Target phrases stripped."
sort -u tempgloss.2.txt > tempgloss.3.txt
awk '{ print length(), $0 | "sort -rn" }' tempgloss.3.txt > tempgloss.4.txt
#echo "List ordered by length."
#echo "Now creating new sed script." # THIS AFFECTS THE SED SCRIPT, NOT THE OUTPUT FILE.
sed '
s/[0-9]//g # strip out all numbers
s/^\ //g # strip leading spaces -- all lines have them due to the sort
/^$/d # delete blank lines
s/\//\\\//g # make text slashes into literals
s/"/\n/g # strip quotes
s/\t//g # strip tabs
s/\./\n/g # strip periods
s/'\''/\\'\''/g # make straight apostrophes into literals
s/'\’'/\\'\’'/g # make curly apostrophes into literals
s/\ /\\ /g # make spaces into literals
/^.\{0,5\}$/d # delete lines with less than five characters
s/^/s\/\\b/ # begins the substitution
s/$/\\b\/\\n\/g/ # concludes the substitution
1 s/^/#!\ \/bin\/sed\ \-f\n\ns\/\[0\-9\]\/\/g\ns\/\\\ \\\ \/\\\ \/g\ns\/\\\.\\\ \/\\n\/g\n\n/
' tempgloss.4.txt > glossy.sed
#echo "glossy.sed created."
chmod +x glossy.sed
echo "Eliminating existing entries. This may take a while."
./glossy.sed $1.0.tmp > $1.1.tmp
echo "Now cleaning up lines."
sed -e '
s/\ $// # strip trailing spaces
s/^\ *//g # strip any and all leading spaces
s/\ el$//g # strip "el" from the end
s/\ la$//g # strip "la" from the end
s/\ los//g # strip "los" from the end
s/\ las//g # strip "las" from the end
s/\ o$//g # strip "o" from the end
s/\ y$//g # strip "y" from the end
s/\ $// # strip trailing spaces (yes, again)
' $1.1.tmp > $1.2.tmp
echo "Creating ngrams."
./ngrams 5 < $1.2.tmp > $1.3.tmp 2> /dev/null
linecount="$(wc -l < $1.3.tmp)"
#echo $linecount "lines."
if [ "$linecount" -gt "1000" ]
then
echo "Eliminating single instances."
sed '/^1\t/d' $1.3.tmp > $1.4.tmp
else
echo "Fewer than 1000 entries, so keeping all."
cp $1.3.tmp $1.4.tmp
fi
sed -e '
s/[0-9]//g # strip out all numbers
s/^\t//g # strip leading tab
s/^\ *//g # strip any and all leading spaces
/^.\{0,7\}$/d # delete lines with less than six characters
s/\ $// # strip trailing spaces (yes, again)
#s/$/\t/ # add in the tab
' $1.4.tmp > $1.csv
echo "Looking for duplicates."
sh ./dedupe $1.csv
wordstxt=$(wc -w < $1)
#echo $wordstxt
wordslist=$(wc -w < $1.csv)
#echo $wordslist
wordspercent=$(echo "scale=4; $wordslist / $wordstxt" |bc -l)
wordspercentage=$(echo "$wordspercent * 100" |bc -l)
after="$(date +%s)"
elapsed_seconds="$(expr $after - $before)"
rate=$(echo "scale=3; $wordstxt / $elapsed_seconds" |bc -l)
echo "Created "$1.csv", with $wordspercentage% left, in" $elapsed_seconds "seconds." #, for an effective rate of" $rate "words per second."
rm tempgloss.*.txt
rm *.tmp
rm glossy.sed