Created
June 28, 2023 13:48
-
-
Save zmxv/b5e954db4953beaf4f5dfd43b8b64175 to your computer and use it in GitHub Desktop.
Solving TEG with RoBERTa
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import re | |
| from transformers import RobertaTokenizer, RobertaForMaskedLM | |
| import torch | |
| reBlank = re.compile(r"\[([^\]]+)\]") | |
| tokenizer = RobertaTokenizer.from_pretrained("roberta-base") | |
| model = RobertaForMaskedLM.from_pretrained('roberta-base') | |
| # puzzles from https://playteg.com | |
| puzzles_original = [ | |
| "[Programmers] don't like [nature] because it has [too] [many] [bugs].", # TEG 51 | |
| "[Conservatives] see [a] [demon] in [Pride] [Month].", # TEG 50 | |
| "Can you tell the difference between [ignorance] and [apathy]? [I] [don't] [know] and [I] [don't] [care].", # TEG 49 | |
| "Don't you [hate] [it] when [someone] [answers] [their] [own] [question]? I do.", # TEG 48 | |
| "How do you [make] [holy] [water]? You [boil] the [hell] out of it.", # TEG 47 | |
| "People are [shocked] when they realize how [bad] [an] [electrician] [I] [am].", # TEG 46 | |
| "I try hard to [childproof] my [house]. The [kids] still [get] [in].", # TEG 45 | |
| "[Kleptomaniacs] don't [get] [my] [jokes] because they like to [take] [things] [literally].", # TEG 44 | |
| "I [wondered] why the [frisbee] kept [getting] [bigger], but then it [hit] [me].", # TEG 43 | |
| "The [rotation] of [Planet] [Earth] really makes [my] [day].", # TEG 42 | |
| ] | |
| puzzles_lite = [ | |
| "Programmers don't like [nature] because it has too many bugs.", # TEG 51 | |
| "Conservatives see a [demon] in Pride Month.", # TEG 50 | |
| "Can you tell the difference between [ignorance] and apathy? I don't know and I don't care.", # TEG 49 | |
| "Don't you hate it when someone [answers] their own question? I do.", # TEG 48 | |
| "How do you make holy water? You boil the [hell] out of it.", # TEG 47 | |
| "People are [shocked] when they realize how bad an electrician I am.", # TEG 46 | |
| "I try hard to [childproof] my house. The kids still get in.", # TEG 45 | |
| "Kleptomaniacs don't get my jokes because they like to [take] things literally.", # TEG 44 | |
| "I wondered why the frisbee kept getting bigger, but then it [hit] me.", # TEG 43 | |
| "The rotation of Planet [Earth] really makes my day.", # TEG 42 | |
| ] | |
| def get_guesses(puzzle): | |
| tokens = tokenizer.encode(puzzle, return_tensors='pt') | |
| with torch.no_grad(): | |
| hidden_state = model(tokens)[0].squeeze() | |
| ret = [] | |
| for mask_index in [mask.item() for mask in (tokens.squeeze() == tokenizer.mask_token_id).nonzero()]: | |
| idx = torch.topk(hidden_state[mask_index], k=10, dim=0)[1] | |
| ret.append([tokenizer.decode(i.item()).strip() for i in idx]) | |
| return ret | |
| def get_score(guesses, answer): | |
| score = 0 | |
| for i, word in enumerate(answer): | |
| score += 1 if word in guesses[i] else 0 | |
| return score | |
| def solve(puzzles): | |
| solved = 0 | |
| blanks = 0 | |
| for i, puzzle in enumerate(puzzles): | |
| print(f'hint {i}: { reBlank.sub("_", puzzle) }') | |
| answer = reBlank.findall(puzzle) | |
| blanks += len(answer) | |
| print(f'answer: { answer }') | |
| guesses = get_guesses(reBlank.sub("<mask>", puzzle)) | |
| print(f'guess: {guesses}') | |
| score = get_score(guesses, answer) | |
| print(f'score: {score}\n') | |
| solved += score | |
| print(f'solved: {solved} / {blanks}') | |
| if __name__ == "__main__": | |
| solve(puzzles_original) | |
| solve(puzzles_lite) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment