Add manual word review script

wordlist/02-custom-lemmatizations.csv

@@ -1,6 +1,13 @@
 WORD1,WORD2
+ABSOLUTE,ABSOLUTELY
+ACCORDING,ACCORDINGLY
+ACCURATE,ACCURATELY
+ACTIVE,ACTIVELY
+ACTUAL,ACTUALLY
 ADD,ADDS
+ADEQUATE,ADEQUATELY
 ADS,ADDS
+ADVERTIZE,ADVERTISE
 AFFECTED,EFFECT
 AFFECT,EFFECT
 AFFECTIONS,AFFECTION
@@ -12,108 +19,288 @@ ANALOGUE,ANALOG
 ANALYSE,ANALYZE
 ANALYSED,ANALYZE
 ANALYSES,ANALYZE
+ANNUAL,ANNUALLY
+APPARENT,APPARENTLY
+APPROXIMATE,APPROXIMATELY
+ARIZE,ARISE
+AUTHORIZE,authorise
+AUTOMATIC,AUTOMATICALLY
 AUX,OX
 BAGGY,BAGGIE
 BE,BEE
 BERRY,BARRY
 BLEW,BLUE
+BODY,BODILY
 BOT,BOUGHT
 BOULDER,BOLDER
+BRIEF,BRIEFLY
 BRINGS,BRING
 BY,BYE
 BYTE,BITE
 CACHE,CASH
 CAPITOL,CAPITAL
+CAREFUL,CAREFULLY
 CENTRE,CENTER
 CENTS,CENT
 CENTS,SENSE
 CERAMICS,CERAMIC
+CERTAIN,CERTAINLY
+CHARACTERIZE,CHARACTERISE
 CHEQUE,CHECK
+CHIEF,CHIEFLY
 CHILE,CHILI
 CHILE,CHILLY
 CHORD,CORD
 CIVILISATION,CIVILIZATION
+CIVILIZE,CIVILISE
+CLEAR,CLEARLY
+CLOSE,CLOSELY
 COLOURLESS,COLORLESS
 COM,CALM
+COMMON,COMMONLY
+COMPARATIVE,COMPARATIVELY
+COMPLETE,COMPLETELY
+CONSIDERAB,CONSIDERABLY
+CONSIDERABLE,CONSIDERABLY
+CONSISTENT,CONSISTENTLY
+CONSTANT,CONSTANTLY
+CONTINUAL,CONTINUALLY
+CONTINUOUS,CONTINUOUSLY
 CORP,CORE
 CORPS,CORE
+CORRECT,CORRECTLY
 COUNSELLOR,COUNSELOR
 CUE,QUEUE
+CURRENT,CURRENTLY
 DAZE,DAY
 DAZED,DAY
 DEAR,DEER
+DEEP,DEEPLY
+DEFINITE,DEFINITELY
+DELIBERATE,DELIBERATELY
 DESSERT,DESERT
 DEW,DO
 DEW,DUE
 DIED,DYED
+DIFFERENT,DIFFERENTLY
+DIRECT,DIRECTLY
+DISTINCT,DISTINCTLY
+EASY,EASILIY
 EFFECTIVE,EFFECT
+EFFECTIVE,EFFECTIVELY
 EFFECTS,EFFECT
+ELDER,ELDERLY
 ELECTRONICS,ELECTRONIC
+EMPHASIZE,EMPHASISE
+ENTIRE,ENTIRELY
+EQUAL,EQUALLY
+ESSENTIAL,ESSENTIALLY
+EVENTUAL,EVENTUALLY
+EVIDENT,EVIDENTLY
+EXACT,EXACTLY
+EXCEEDING,EXCEEDINGLY
+EXCLUSIVE,EXCLUSIVELY
+EXERCIZE,EXERCISE
+EXPLICIT,EXPLICITLY
+EXTENSIVE,EXTENSIVELY
+EXTREME,EXTREMELY
+FAIR,FAIRLY
 FAVOUR,FAVOR
 FAX,FACTS
 FILING,FILLING
 FILINGS,FILLING
+FINAL,FINALLY
+FIRM,FIRMLY
+FORMAL,FORMALLY
+FORMER,FORMERLY
 FORMULAE,FORMULA
+FORTUNATE,FORTUNATELY
 FORTUNATELY,FORTUNATE
 FOUR,FOR
+FREE,FREELY
+FREQUENT,FREQUENTLY
+FRIEND,FRIENDLY
+FULL,FULLY
+GENERAL,GENERALLY
+GENERALIZE,GENERALISE
+GENTLE,GENTLY
+GRADUAL,GRADUALLY
 GRATE,GREAT
+GREAT,GREATLY
 HAIRY,HARRY
+HARD,HARDLY
 HARRY,HAIRY
+HEAVEN,HEAVENLY
 HEIR,HAIR
 HEIRS,HAIR
 HEM,HIM
+HIGH,HIGHLY
 HONOUR,HONOR
 HONOURS,HONORS
 HYMN,HIM
 HYMNS,HIM
+IMMEDIATE,IMMEDIATELY
 IMPROVES,IMPROVE
+INCREASING,INCREASINGLY
+INDIRECT,INDIRECTLY
+INEVITABLE,INEVITABLY
+INITIAL,INITIALLY
+INSTANT,INSTANTLY
+INVARIABLE,INVARIABLY
 ISLE,AISLE
+KIND,KINDLY
 KNIGHT,NIGHT
 KNOT,NOT
 KNOTS,NOT
+LARGE,LARGELY
 LARVAE,LARVA
 LECTURER,LECTURE
+LIGHT,LIGHTLY
+LIKE,LIKELY
+LITERAL,LITERALLY
+LIVE,LIVELY
+LOCALIZE,LOCALISE
+LOVE,LOVELY
+MAIN,MAINLY
 MANOR,MANNER
+MERE,MERELY
 MONIES,MONEYS
+MONTH,MONTHLY
+MOST,MOSTLY
+NAME,NAMELY
+NATURAL,NATURALLY
+NEAR,NEARLY
+NECESSARY,NECESSARILY
 NEIGHBOURHOOD,NEIGHBORHOOD
 NEIGHBOUR,NEIGHBOR
 NEIGHBOURS,NEIGHBOR
+NEW,NEWLY
+NORMAL,NORMALLY
 NOSE,KNOW
+NOTABLE,NOTABLY
 NUN,NONE
+OBVIOUS,OBVIOUSLY
+OCCASIONAL,OCCASIONALLY
+OPEN,OPENLY
 ORE,OAR
 ORE,OR
 ORGANISATIONAL,ORGANIZATIONAL
 ORGANISATION,ORGANIZATION
 ORGANISATIONS,ORGANIZATION
+ORIGINAL,ORIGINALLY
 OWE,OH
 PAR,PARSE
 PARS,PARSE
+PARTIAL,PARTIALLY
+PARTICULAR,PARTICULARLY
+PART,PARTLY
 PEOPLES,PEOPLE
+PERFECT,PERFECTLY
+PERMANENT,PERMANENTLY
 PER,PURR
+PERSONAL,PERSONALLY
 PETAL,PEDAL
+PHYSICAL,PHYSICALLY
+PLAIN,PLAINLY
+POLITICAL,POLITICALLY
 POLLING,PULLING
+POSITIVE,POSITIVELY
+POSSIBLE,POSSIBLY
+POTENTIAL,POTENTIALLY
+PRACTICAL,PRACTICALLY
+PRECISE,PRECISELY
+PRESENT,PRESENTLY
+PRESUMAB,PRESUMABLY
+PREVIOUS,PREVIOUSLY
+PRINCIPAL,PRINCIPALLY
+PROBABLE,PROBABLY
+PROMPT,PROMPTLY
+PROPER,PROPERLY
 PROVIDES,PROVIDE
+PUBLIC,PUBLICLY
+PURE,PURELY
+QUICK,QUICKLY
+QUIET,QUIETLY
+RAPID,RAPIDLY
 RAP,WRAP
+RARE,RARELY
+REALIZE,REALISE
+REAL,REALLY
+REASONABLE,REASONABLY
+RECENT,RECENTLY
 REFORMED,REFORM
+REGULAR,REGULARLY
+RELATIVE,RELATIVELY
 REPEATEDLY,REPEAT
 REPEATED,REPEAT
+REPEATED,REPEATEDLY
+RESPECTIVE,RESPECTIVELY
+RIGHT,RIGHTLY
+ROUGH,ROUGHLY
+SAFE,SAFELY
+SCARCE,SCARCELY
 SCENT,CENT
 SCENTS,CENT
+SEEMING,SEEMINGLY
 SENSE,CENT
 SENSED,CENT
 SENSES,CENT
+SEPARATE,SEPARATELY
+SERIOUS,SERIOUSLY
+SEVERE,SEVERELY
+SHARP,SHARPLY
+SHORT,SHORTLY
+SIGNIFICANT,SIGNIFICANTLY
+SIMILAR,SIMILARLY
+SIMPLE,SIMPLY
 SIMULTANEOUSLY,SIMULTANEOUS
+SIMULTANEOUS,SIMULTANEOUSLY
+SLIGHT,SLIGHTLY
+SLOW,SLOWLY
+SOCIAL,SOCIALLY
+SOLE,SOLELY
+SPECIALIZE,SPECIALISE
+SPECIAL,SPECIALLY
+STEADY,STEADILY
+STRICT,STRICTLY
+STRONG,STRONGLY
+SUBSEQUENT,SUBSEQUENTLY
+SUBSTANTIAL,SUBSTANTIALLY
+SUCCESSFUL,SUCCESSFULLY
+SUDDEN,SUDDENLY
+SUFFICIENT,SUFFICIENTLY
+SUMMARIZE,SUMMARISE
+SURE,SURELY
+SURPRIZE,SURPRISE
 TELECOMMUNICATIONS,TELECOMMUNICATION
+TEMPORARI,TEMPORARILY
+TEMPORARY,TEMPORARILY
 THEATRES,THEATER
 THEATRE,THEATER
 THIN,THEN
+THOROUGH,THOROUGHLY
 THRU,THROUGH
+TOTAL,TOTALLY
+TRUE,TRULY
+TYPICAL,TYPICALLY
+ULTIMATE,ULTIMATELY
+UNDOUBTED,UNDOUBTEDLY
+UNLIKE,UNLIKELY
 UNTO,ONTO
+USUAL,USUALLY
+UTILIZE,UTILISE
+UTTER,UTTERLY
 VAPOUR,VAPOR
 VARY,VERY
 VERTEBRA,VERTEBRAE
+VIRTUAL,VIRTUALLY
 WEARY,WARY
+WEEK,WEEKLY
 WEIGHS,WAY
 WEIGH,WAY
+WIDE,WIDELY
 YELLOW,HELLO
 YOKE,YOLK
+ALLOCATE,ALLOCATION
+ANYBODY,ANYONE
+BESIDE,BESIDES

wordlist/03-exclude.csv

@@ -3897,3 +3897,23 @@ ZOOLOGY
 ZOOT-SUIT
 ZULU
 ZURICH
+SWORD
+TREMBLE
+BAD
+PAIN
+ABNORMAL
+ABSENCE
+ACADEMICS
+ACCENT
+ACCUSTOM
+ACID
+ACQUISITION
+ABSORPTION
+ACCEPTABLE
+ACCEPTANCE
+ACTIVATION
+SCARCE
+FOLLY
+MELANCHOLY
+PRACTIZE
+PREMIZE

wordlist/05-accepted-words.txt

@@ -0,0 +1,356 @@
+ABIDE
+ABILITY
+ABLE
+ABOUT
+ABOVE
+ABROAD
+ABSOLUTE
+ABSORB
+ABSTRACT
+ABUNDANCE
+ABUNDANT
+ACCELERATE
+ACCEPT
+ACCESS
+ACCESSIBLE
+ACCOMMODATE
+ACCOMPANY
+ACCOMPLISH
+ACCOMPLISHMENT
+ACCORD
+ACCORDANCE
+ACCOUNT
+ACCUMULATE
+ACCUMULATION
+ACCURACY
+ACCURATE
+ACHIEVE
+ACHIEVEMENT
+ACKNOWLEDGE
+ACQUAINT
+ACQUAINTANCE
+ACQUIRE
+ACRE
+ACROSS
+ACT
+ACTION
+ACTIVATE
+ACTIVE
+ACTIVITY
+ACTOR
+ACTUAL
+ABSOLUTELY
+ACCORDING
+ACUTE
+ADAPT
+ADAPTATION
+ADDITION
+ADDITIONAL
+ADDRESS
+ADHERE
+ADJACENT
+ADJUST
+ADJUSTMENT
+ADMINISTER
+ADMINISTRATION
+ADMINISTRATIVE
+ADMINISTRATOR
+ADMIRABLE
+ADMIRATION
+ADMIRE
+ADMISSION
+ADMIT
+ADOPT
+ADOPTION
+ADVANCE
+ADVANTAGE
+ADVENTURE
+ADVERTISEMENT
+ADVERTIZE
+ADVICE
+ADVISER
+ADVOCATE
+AFFINITY
+AFFIRM
+AFFORD
+AFTER
+AFTERNOON
+AFTERWARDS
+AGAIN
+AGENCY
+AGENT
+AGGREGATE
+AGO
+AGREE
+AGREEABLE
+AGREEMENT
+AGRICULTURAL
+AGRICULTURE
+AIR
+AIRCRAFT
+ALGORITHM
+ALIKE
+ALIVE
+ALL
+ALLEGE
+ALLIANCE
+ALLOCATE
+ALLOCATION
+ALLOW
+ALLOWANCE
+ALLOY
+ALLUDE
+ALLUSION
+ALLY
+ALMOST
+ALONG
+ALONGSIDE
+ALREADY
+ALSO
+ALTER
+ALTERATION
+ALTERNATE
+ALTERNATIVE
+ALTHOUGH
+ALTITUDE
+ALTOGETHER
+ALWAYS
+AMAZE
+AMBITION
+AMBITIOUS
+AMEND
+AMENDMENT
+AMID
+AMONG
+AMONGST
+AMOUNT
+AMPLE
+AMPLITUDE
+AMUSE
+AMUSEMENT
+ANALOGOUS
+ANALOGY
+ANALYSIS
+ANALYST
+ANALYTICAL
+ANCHOR
+ANCIENT
+ANGEL
+ANGLE
+ANGULAR
+ANIMATE
+ANNOUNCE
+ANNOUNCEMENT
+ANOTHER
+ANSWER
+ANTERIOR
+ANTICIPATE
+ANY
+ANYBODY
+ANYONE
+ANYTHING
+ANYWAY
+ANYWHERE
+APART
+APARTMENT
+APOSTLE
+APPARATUS
+APPEALING
+APPEAR
+APPEARANCE
+APPETITE
+APPLE
+APPLICABLE
+APPLICANT
+APPLICATION
+APPLY
+APPOINT
+APPOINTMENT
+APPRECIATE
+APPRECIATION
+APPROACH
+APPROVAL
+APPROVE
+APPROXIMATION
+APT
+ARBITRARY
+ARC
+ARCH
+ARCHITECT
+ARCHITECTURAL
+ARE
+AREA
+ARM
+ARMOR
+AROUND
+ARRANGE
+ARRANGEMENT
+ARRAY
+ARRIVAL
+ARRIVE
+ARROW
+ART
+ARTERIAL
+ARTERY
+ARTICLE
+ARTICULATE
+ARTIST
+ARTISTIC
+ASCEND
+ASCERTAIN
+ASCRIBE
+ASIDE
+ASK
+ASLEEP
+ASPECT
+ASPIRATION
+ASSEMBLE
+ASSERT
+ASSERTION
+ASSESS
+ASSESSMENT
+ASSET
+ASSIGN
+ASSIGNMENT
+ASSIST
+ASSISTANCE
+ASSISTANT
+ASSOCIATE
+ASSOCIATION
+ASSUME
+ASSUMPTION
+ASSURANCE
+ASSURE
+ASTONISH
+AT
+ATMOSPHERE
+ATOM
+ATTACH
+ATTACHMENT
+ATTAIN
+ATTAINMENT
+ATTEMPT
+ATTEND
+ATTENDANCE
+ATTENDANT
+ATTENTION
+ATTITUDE
+ATTORNEY
+ATTRACT
+ATTRIBUTE
+AUDIENCE
+AUNT
+AUTHENTIC
+AUTHOR
+AUTHORITY
+AUTOMOBILE
+AUTUMN
+AVAIL
+AVAILABILITY
+AVAILABLE
+AVERAGE
+AWAIT
+AWAKE
+AWAKEN
+AWARD
+AWARE
+AWARENESS
+AWAY
+BACK
+BACKGROUND
+BAG
+BALANCE
+BALL
+BAND
+BANK
+BARK
+BARREL
+BASE
+BASIC
+BASICALLY
+BASIN
+BASIS
+BASKET
+BATTERY
+BAY
+BEACH
+BEAD
+BEAM
+BEAR
+BEARD
+BEAUTIFUL
+BEAUTY
+BECAUSE
+BECOME
+BED
+BEDROOM
+BEER
+BEFORE
+BEGIN
+BEHALF
+BEHAVE
+BEHAVIOR
+BEHAVIOURAL
+BEHOLD
+BEING
+BELL
+BELONG
+BELOVED
+BELOW
+BENCH
+BEND
+BENEFIT
+BENT
+BESIDE
+BESIDES
+BEST
+BESTOW
+BET
+BETWEEN
+BEYOND
+BID
+BIG
+BILL
+BILLION
+BIOGRAPHY
+BIOLOGICAL
+BIRD
+BIRTH
+BIT
+BLANKET
+BLOOM
+BOAT
+BOLD
+BOLT
+BOND
+BOOK
+BOOT
+BORDER
+BORROW
+BOTH
+BOTTLE
+BOTTOM
+BOUND
+BOUNDARY
+BOW
+BOWL
+BOX
+BRAIN
+BRANCH
+BRAND
+BRASS
+BRAVE
+BREAD
+BREADTH
+BREAKFAST
+BREATH
+BREATHE
+BRICK
+BRIDGE
+BRIGHT
+BRILLIANT
+BROADCAST
+BRONZE
+BROW
+BRUSH
+BUBBLE

wordlist/05-interactive-sorting.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -euo pipefail
+# set -x
+
+ACCEPTED=./05-accepted-words.txt
+REJECTED=./05-rejected-words.txt
+INFILE=./04-deduplicated-words.csv
+
+get_char() {
+    local escape_char="$(printf "\u1b")" mode
+    while true; do
+        read -rsn1 mode
+        if [[ $mode == $escape_char ]]; then
+            read -rsn2 mode # read 2 more chars
+        fi
+        case "$mode" in
+            'q') return 1;;
+            # '[A') echo UP ;;
+            # '[B') echo DOWN;;
+            '[D') echo 0; return 0;; # Reject (left)
+            '[C') echo 1; return 0;; # Keep (right)
+            *) :;;
+        esac
+    done
+}
+
+set -o noglob
+IFS=$'\n'
+ALL_WORDS=($(awk -F, 'NR!=1&&!a[$2]++ {print $1}' "${INFILE}" | sort -u))
+
+touch -- "${ACCEPTED}" "${REJECTED}"
+for WORD in "${ALL_WORDS[@]}"; do
+    if grep -xqFe "${WORD}" "${ACCEPTED}" "${REJECTED}" <(awk -F, 'NR!=1{print $1 "\n" $2}' 02-custom-lemmatizations.csv); then
+        continue
+    fi
+
+    printf '%s\n' "${WORD}"
+
+    KEEP="$(get_char)"
+
+    if ((KEEP)); then
+        printf '%s\n' "${WORD}" >>"${ACCEPTED}"
+    else
+        printf '%s\n' "${WORD}" >>"${REJECTED}"
+    fi
+done

wordlist/05-rejected-words.txt

@@ -0,0 +1,48 @@
+
+AFRAID
+AGAINST
+AGENDUM
+AHEAD
+AIM
+ALARM
+ALERT
+ALONE
+ANTI
+ANTIBODY
+ANGER
+ANIMAL
+APPROPRIATE
+ARCHITECTURE
+ARMY
+AROUSE
+ARTIFICIAL
+ASH
+ASHAMED
+AVOID
+AXIS
+BACTERIAL
+BACTERIUM
+BARE
+BARELY
+BATH
+BATTLE
+BELIEVE
+BELIEVER
+BENEFICIAL
+BETRAY
+BIBLICAL
+BLADE
+BLANK
+BLAST
+BLEND
+BLESS
+BLIND
+BLOCK
+BLOW
+BOAST
+BOSS
+BREAKDOWN
+BREED
+BRIDE
+BROAD
+BROWN