{"nbformat":4,"nbformat_minor":0,"metadata":{"accelerator":"GPU","colab":{"provenance":[{"file_id":"1r_eoi8CMea_a3YjWC1M4EmTqKMGVMbzQ","timestamp":1654373253252},{"file_id":"18SKxCwdiWqcX4wXjLMpBDSzMuoZ2Sd1I","timestamp":1619109434941},{"file_id":"1Qsxpdye-9_uBRMMfJkLw28-LMMHqXoEI","timestamp":1616631420523},{"file_id":"1IH5LdMWCHYreOhUKdkOmvQCYz-H9nkDd","timestamp":1616535622996}]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"6ea5836ea6fa4b169afc45e4552e88d3":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_65dddfc7348f47319111ecfcfe7b55a6","IPY_MODEL_d90615a77d154c9390d4321355b0da55","IPY_MODEL_ad8c39455fdc4c6492cbddbdb968a3ea"],"layout":"IPY_MODEL_e1b980b79c554aef8c433a2c758efe9a"}},"65dddfc7348f47319111ecfcfe7b55a6":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_c93ed4057b4f4c07aec4a338106ebdb6","placeholder":"​","style":"IPY_MODEL_ced2c99134e64da580308c6f1414a413","value":"Downloading (…)okenizer_config.json: 100%"}},"d90615a77d154c9390d4321355b0da55":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_0f9574c4b5ec43aa8c1101b0d1d10897","max":28,"min":0,"orientation":"horizontal","style":"IPY_MODEL_554e9ca8a6ad443fa360d9dcf5ba6671","value":28}},"ad8c39455fdc4c6492cbddbdb968a3ea":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ab53d4da31284b358cf0d36e73ce96f5","placeholder":"​","style":"IPY_MODEL_8a191d081b6b428b8d7f1f1912c47d09","value":" 28.0/28.0 [00:00<00:00, 539B/s]"}},"e1b980b79c554aef8c433a2c758efe9a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c93ed4057b4f4c07aec4a338106ebdb6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ced2c99134e64da580308c6f1414a413":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0f9574c4b5ec43aa8c1101b0d1d10897":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"554e9ca8a6ad443fa360d9dcf5ba6671":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"ab53d4da31284b358cf0d36e73ce96f5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8a191d081b6b428b8d7f1f1912c47d09":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a66c1ca228f947018b5109d4b90e6742":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_428e73f4b3684b2d9749efdcc03c399c","IPY_MODEL_82832e67726e4c2e8a61d81e5e81cdbc","IPY_MODEL_d0907d18f5d54e839a33ede7ea2c671c"],"layout":"IPY_MODEL_4b4f3c035f76495baf8df4a3cccc4ca2"}},"428e73f4b3684b2d9749efdcc03c399c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0583940dd1d7447f95ca00d0106d35b6","placeholder":"​","style":"IPY_MODEL_52e17f09dfa2455796031a54e30c3182","value":"Downloading (…)solve/main/vocab.txt: 100%"}},"82832e67726e4c2e8a61d81e5e81cdbc":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_77f91290cd9b434ca667a53d6cd5df8f","max":231508,"min":0,"orientation":"horizontal","style":"IPY_MODEL_b3d6c2f5ed73410a81da1c5e8f1837dc","value":231508}},"d0907d18f5d54e839a33ede7ea2c671c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d72b4c04eadc4ae69d4e625080acb934","placeholder":"​","style":"IPY_MODEL_4cb3c4601b39408388add9a2ccb3f217","value":" 232k/232k [00:00<00:00, 2.10MB/s]"}},"4b4f3c035f76495baf8df4a3cccc4ca2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0583940dd1d7447f95ca00d0106d35b6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"52e17f09dfa2455796031a54e30c3182":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"77f91290cd9b434ca667a53d6cd5df8f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b3d6c2f5ed73410a81da1c5e8f1837dc":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"d72b4c04eadc4ae69d4e625080acb934":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4cb3c4601b39408388add9a2ccb3f217":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a2df16cd32104e09bb024d516197578c":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_16b949707d1446ec8d2cbbcbb56ad1dc","IPY_MODEL_59788bf989c842d4b58ff02aee66154f","IPY_MODEL_35f3660dbceb477ebb1c242b73573c6b"],"layout":"IPY_MODEL_38ebf0b9f040488a9ce9fb8a1b320386"}},"16b949707d1446ec8d2cbbcbb56ad1dc":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f933cbeeed3045609e7721adc4bba32c","placeholder":"​","style":"IPY_MODEL_4edf735ed9ed4fd99f5c6474e025b4a7","value":"Downloading (…)/main/tokenizer.json: 100%"}},"59788bf989c842d4b58ff02aee66154f":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_7046d1cd0e704d90904af8a278df9aef","max":466062,"min":0,"orientation":"horizontal","style":"IPY_MODEL_0c0f37a4160c41059277b61bc412aa56","value":466062}},"35f3660dbceb477ebb1c242b73573c6b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_db46befff00d4388925702c8e9e8e2d7","placeholder":"​","style":"IPY_MODEL_30e26faca8764512b7c5ec4d050c63d3","value":" 466k/466k [00:00<00:00, 5.98MB/s]"}},"38ebf0b9f040488a9ce9fb8a1b320386":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f933cbeeed3045609e7721adc4bba32c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4edf735ed9ed4fd99f5c6474e025b4a7":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"7046d1cd0e704d90904af8a278df9aef":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0c0f37a4160c41059277b61bc412aa56":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"db46befff00d4388925702c8e9e8e2d7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"30e26faca8764512b7c5ec4d050c63d3":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"afe754ece46340e1bc66149515dbd22f":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_2d812091561b4e4b983e2bbc2eab7ea1","IPY_MODEL_a45bfcf8685144c18e6934caeef9e59e","IPY_MODEL_3f345e030ab3489bb3c3008d4bbe423c"],"layout":"IPY_MODEL_415dba2a63244f868451dc4245ad68d2"}},"2d812091561b4e4b983e2bbc2eab7ea1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_792a2480c3d5493586c1fb88ce1c333f","placeholder":"​","style":"IPY_MODEL_e7ed2cf27fe44c9abdf6111d0b716951","value":"Downloading (…)lve/main/config.json: 100%"}},"a45bfcf8685144c18e6934caeef9e59e":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_dd1ab8f05e5b4168aaac7d92f4c0ed33","max":483,"min":0,"orientation":"horizontal","style":"IPY_MODEL_8f0bf5b6d5954383b8ad9c5c3fb444ab","value":483}},"3f345e030ab3489bb3c3008d4bbe423c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_913bf28f61b94769a3b14c0f08368535","placeholder":"​","style":"IPY_MODEL_1d6d270d4da149ab8ebbcdafb2765218","value":" 483/483 [00:00<00:00, 10.6kB/s]"}},"415dba2a63244f868451dc4245ad68d2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"792a2480c3d5493586c1fb88ce1c333f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e7ed2cf27fe44c9abdf6111d0b716951":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"dd1ab8f05e5b4168aaac7d92f4c0ed33":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8f0bf5b6d5954383b8ad9c5c3fb444ab":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"913bf28f61b94769a3b14c0f08368535":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1d6d270d4da149ab8ebbcdafb2765218":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"5af49b4dad9a46bda97e746f3745cf97":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_dc95e652a1f944e88d685c7993c63794","IPY_MODEL_38571b5f77fe462e9fcefea391347d98","IPY_MODEL_0260f68e741b462db70933abee84083b"],"layout":"IPY_MODEL_efb1729e64ad44dda977a52fbdf7f6ec"}},"dc95e652a1f944e88d685c7993c63794":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_673de86e2d1a48d484654ccf93a0bb60","placeholder":"​","style":"IPY_MODEL_051d5a95aee8408ea0059467151e64b9","value":"Downloading pytorch_model.bin: 100%"}},"38571b5f77fe462e9fcefea391347d98":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_bd20f60eb60543f0880c74d143e32135","max":267967963,"min":0,"orientation":"horizontal","style":"IPY_MODEL_58cdaea6a49e4d099fae61986386a840","value":267967963}},"0260f68e741b462db70933abee84083b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7afc0329bc1742e8993596450a1de678","placeholder":"​","style":"IPY_MODEL_8f0baa6e3eed478d8148acccac0da4e5","value":" 268M/268M [00:03<00:00, 79.3MB/s]"}},"efb1729e64ad44dda977a52fbdf7f6ec":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"673de86e2d1a48d484654ccf93a0bb60":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"051d5a95aee8408ea0059467151e64b9":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"bd20f60eb60543f0880c74d143e32135":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"58cdaea6a49e4d099fae61986386a840":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"7afc0329bc1742e8993596450a1de678":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8f0baa6e3eed478d8148acccac0da4e5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"cells":[{"cell_type":"markdown","metadata":{"id":"ucj6rr9Z7gt3"},"source":["# Measuring Word Similarity with BERT (Sephora Makeup Reviews)\n","\n","By [The BERT for Humanists](https://melaniewalsh.github.io/BERT-for-Humanists/) Team\n","\n","How can we measure the similarity of words, or word uses, in a collection of texts? Let's say we're interested in a collection of Sephora makeup reviews, and we specifically want to understand the experiences of customers who are \"sensitive\" to makeup in various ways. Do reviewers use the word \"sensitive\" similarly or differently when describing different products, or when rating a product positively or negatively? What about when reviewers discuss how well makeup holds up at the the \"pool, the \"gym,\" the \"office, or a \"wedding\" — how do these contexts compare? \n","\n","We can explore all of these questions with BERT, a natural language processing model that has revolutionized the field.\n","\n","BERT turns words or tokens into vectors — essentially, a list of numbers in a coordinate system (x, y). We can then use the geometric similarity between these resulting vectors as a way to represent varying types of similarity between words.\n","\n","## In This Notebook\n","In this Colab notebook, we will specifically analyze a collection of 5k [Sephora makeup reviews](https://github.com/everestpipkin/datagardens/tree/master/students/khanniie/5_newDataSet) scraped by Google engineer [Connie Ye](https://connieye.com/about). While an undergraduate student at Carnegie Mellon, Ye completed a project about Sephora reviews that specifically mentioned crying, and she even created a [website](https://connie.dog/sephora/) where you can explore these waterlogged reviews.\n","\n","For our purposes here, we will analyze all 5k Sephora reviews with the [DistilBert model](https://huggingface.co/transformers/model_doc/distilbert.html) and the HuggingFace Python library. DistilBert is a smaller — yet still powerful! — version of BERT. By using the rich representations of words that BERT produces, we will then explore the multivalent meanings of particular words in context.\n","\n","We hope this notebook will help illustrate how BERT works, how well it works, and how you might use BERT to explore the similarity of words in a collection of texts. But we also hope that these results will expose some of the limitations and challenges of BERT."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":585},"id":"zsAaAlwhZsKr","executionInfo":{"status":"ok","timestamp":1654525821132,"user_tz":240,"elapsed":373,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"89602aad-c6f1-4864-aba3-d37a8059fec1"},"source":["#@title BERT Word Vectors: A Preview { display-mode: \"form\" }\n","#@title: Hover\n","import pandas as pd\n","import altair as alt\n","\n","url = \"https://github.com/melaniewalsh/Neat-Datasets/raw/main/bert-word-sensitive.csv\"\n","df = pd.read_csv(url, encoding='utf-8')\n","\n","search_keywords = ['sensitive']\n","color_by = 'word'\n","\n","alt.Chart(df, title=f\"Word Similarity in Sephora Reviews: {', '.join(search_keywords).title()}\").mark_circle(size=200).encode(\n"," alt.X('x',\n"," scale=alt.Scale(zero=False)\n"," ), y=\"y\",\n"," color= color_by,\n"," tooltip=['word', 'context', 'type', 'brand']\n"," ).interactive().properties(\n"," width=500,\n"," height=500\n",")"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["\n","
\n",""],"text/plain":["alt.Chart(...)"]},"metadata":{},"execution_count":59}]},{"cell_type":"markdown","metadata":{"id":"4yycYqe4W-MI"},"source":["The plot above displays a preview of our later results. This is what we're working toward!\n","\n","You can hover over each point to see the instance of each word in context."]},{"cell_type":"markdown","metadata":{"id":"y73xWUkpU1AC"},"source":["



\n","\n","## **Import necessary Python libraries and modules**"]},{"cell_type":"markdown","metadata":{"id":"4N0RIalt28yL"},"source":["Ok enough introduction! Let's get started.\n","\n","To use the HuggingFace [`transformers` Python library](https://huggingface.co/transformers/installation.html), we first need to install it with `pip`."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"mz4lRqP0umyq","executionInfo":{"status":"ok","timestamp":1684782599030,"user_tz":420,"elapsed":25884,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"5a82ef1c-fa2e-4bf1-a43e-0511ee6f5ff4"},"source":["!pip install transformers"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting transformers\n"," Downloading transformers-4.29.2-py3-none-any.whl (7.1 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m29.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n","Collecting huggingface-hub<1.0,>=0.14.1 (from transformers)\n"," Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n","Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n"," Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m33.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n","Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.4.0)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.5.0)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n","Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n","Installing collected packages: tokenizers, huggingface-hub, transformers\n","Successfully installed huggingface-hub-0.14.1 tokenizers-0.13.3 transformers-4.29.2\n"]}]},{"cell_type":"markdown","metadata":{"id":"BJ9kmfrO3M_w"},"source":["Then we will import the DistilBertModel and DistilBertTokenizerFast from the Hugging Face `transformers` library. We will also import a handful of other Python libraries and modules."]},{"cell_type":"code","metadata":{"id":"6lcEPhcuV9f5"},"source":["# For BERT\n","from transformers import DistilBertTokenizerFast, DistilBertModel\n","\n","# For data manipulation and analysis\n","import pandas as pd\n","pd.options.display.max_colwidth = 200\n","import numpy as np\n","from sklearn.decomposition import PCA\n","\n","# For interactive data visualization\n","import altair as alt"],"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from collections import defaultdict\n","import random\n","import json\n","from urllib.request import urlopen"],"metadata":{"id":"fMx098zNy50f"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"P_C4hiuqvt5L"},"source":["



\n","\n","## **Load text dataset**"]},{"cell_type":"markdown","metadata":{"id":"KwZzU8bA2MU-"},"source":["Below we will read in a number of JSON files that contain the Sephora reviews. This is simply how we've chosen to load our data, and any way that you are comfortable lodaing data should work for this step. In the end, all you really need is a list of texts."]},{"cell_type":"code","source":["# dataset_url = 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/crying_dataset.json'\n","\n","dataset_urls = ['https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/better_than_sex.json',\n"," 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/better_than_sex_waterproof.json',\n"," 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/cannonball.json',\n"," 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/diorshow_waterproof.json',\n"," 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/emotionproof.json',\n"," 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/fenty.json',\n"," 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/hangover_primer.json',\n"," 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/highliner.json',\n"," 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/kat_von_d_tattoo_liner.json',\n"," # 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/kvd_inkwell_longwear.json',\n"," 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/primer.json',\n"," 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/stila_waterproof_eyeliner.json',\n"," 'https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/urban_decay_setting_spray.json']"],"metadata":{"id":"lURLLv6M92IF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Read in each dataset and merged together\n","df_list = []\n","for _url in dataset_urls:\n"," # _json = pd.read_json(_url)\n"," print(_url)\n"," _df = pd.DataFrame(pd.read_json(_url)['reviews'].tolist())\n"," _json = json.loads(urlopen(_url).read())\n"," _df['brand'] = _json['brand']\n"," _df['name'] = _json['product_name']\n"," _df['type'] = _json['product_type']\n"," _df['url'] = _json['url']\n"," df_list.append(_df)\n","\n","reviews_df = pd.concat(df_list)\n","\n","len(reviews_df.index)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"IiuRcz9sx9Eg","executionInfo":{"status":"ok","timestamp":1684782613615,"user_tz":420,"elapsed":4362,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"af9336c9-4a5d-4b67-a9b3-347eedba697e"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/better_than_sex.json\n","https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/better_than_sex_waterproof.json\n","https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/cannonball.json\n","https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/diorshow_waterproof.json\n","https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/emotionproof.json\n","https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/fenty.json\n","https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/hangover_primer.json\n","https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/highliner.json\n","https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/kat_von_d_tattoo_liner.json\n","https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/primer.json\n","https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/stila_waterproof_eyeliner.json\n","https://raw.githubusercontent.com/everestpipkin/datagardens/master/students/khanniie/5_newDataSet/all_json/urban_decay_setting_spray.json\n"]},{"output_type":"execute_result","data":{"text/plain":["5018"]},"metadata":{},"execution_count":5}]},{"cell_type":"markdown","source":["Let's look at a sample of reviews"],"metadata":{"id":"8-vlqN5513iN"}},{"cell_type":"code","source":["reviews_df.sample(3)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":274},"id":"RLiI_QZy-G8R","executionInfo":{"status":"ok","timestamp":1684782613616,"user_tz":420,"elapsed":21,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"2b493d55-6336-4483-f5c6-16f68dc812fb"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":[" extra_info stars date \\\n","307 Age 25-34 2 stars 15 Mar 2016 \n","180 Age 25-34 5 stars 1 Aug 2017 \n","86 error getting this field. 5 stars 29 Apr 2016 \n","\n"," title \\\n","307 Patchy \n","180 only buy the travel size \n","86 Best Eyeliner i hav ever used! \n","\n"," description \\\n","307 At first, this applied like a dream. \\r\\nI have very oily hooded lids and I think that may have done something to the formula. \\r\\nBut at first application freshly opened, it glided on smoothly, w... \n","180 I only buy the travel size because the brush is better and the formula has less air in the container to make the product go bad. HOLY GRAIL \n","86 I got the sample size in my birthday gift and I was really skeptical. I absolutely hate crayon eyeliner because it always smears and fades, but when I swatched this on my hand I was amazed. The bl... \n","\n"," brand name type \\\n","307 Kat Von D Tattoo Liner Liquid Eyeliner \n","180 Too Faced Better Than Sex Mascara Mascara \n","86 Marc Jacobs Highliner Gel Eye Crayon Eyeliner Gel Eyeliner \n","\n"," url \n","307 https://www.sephora.com/product/tattoo-liner-P245205?icid2=products%20grid:p245205 \n","180 https://www.sephora.com/product/better-than-sex-mascara-P381000?icid2=products%20grid:p381000 \n","86 https://www.sephora.com/product/highliner-gel-crayon-P379434?icid2=products%20grid:p379434 "],"text/html":["\n","
\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
extra_infostarsdatetitledescriptionbrandnametypeurl
307<b>Age</b> 25-342 stars15 Mar 2016PatchyAt first, this applied like a dream. \\r\\nI have very oily hooded lids and I think that may have done something to the formula. \\r\\nBut at first application freshly opened, it glided on smoothly, w...Kat Von DTattoo LinerLiquid Eyelinerhttps://www.sephora.com/product/tattoo-liner-P245205?icid2=products%20grid:p245205
180<b>Age</b> 25-345 stars1 Aug 2017only buy the travel sizeI only buy the travel size because the brush is better and the formula has less air in the container to make the product go bad. HOLY GRAILToo FacedBetter Than Sex MascaraMascarahttps://www.sephora.com/product/better-than-sex-mascara-P381000?icid2=products%20grid:p381000
86error getting this field.5 stars29 Apr 2016Best Eyeliner i hav ever used!I got the sample size in my birthday gift and I was really skeptical. I absolutely hate crayon eyeliner because it always smears and fades, but when I swatched this on my hand I was amazed. The bl...Marc JacobsHighliner Gel Eye Crayon EyelinerGel Eyelinerhttps://www.sephora.com/product/highliner-gel-crayon-P379434?icid2=products%20grid:p379434
\n","
\n"," \n"," \n"," \n","\n"," \n","
\n","
\n"," "]},"metadata":{},"execution_count":6}]},{"cell_type":"markdown","source":["What types of makeup are being reviewed?"],"metadata":{"id":"h0jXM2a4bRoM"}},{"cell_type":"code","source":["reviews_df['type'].value_counts()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"zK290aGBV2g3","executionInfo":{"status":"ok","timestamp":1684782613616,"user_tz":420,"elapsed":18,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"f4c2c336-dcbd-4b5f-8717-168002e89e6a"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Mascara 1786\n","Liquid Eyeliner 1535\n","Gel Eyeliner 584\n","Primer 403\n","Setting Spray 391\n","Eye Primer 319\n","Name: type, dtype: int64"]},"metadata":{},"execution_count":7}]},{"cell_type":"markdown","source":["What brands are being reviewed?"],"metadata":{"id":"DzArhXJYbTYr"}},{"cell_type":"code","source":["reviews_df['brand'].value_counts()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"k-KN33cC-G-m","executionInfo":{"status":"ok","timestamp":1684782613616,"user_tz":420,"elapsed":15,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"a50dee3a-51e4-456b-b78f-e0934498e31b"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Too Faced 1373\n","Urban Decay 1033\n","Marc Jacobs 584\n","Kat Von D 567\n","FENTY BEAUTY BY RIHANNA 550\n","Stila 418\n","Dior 407\n","Tom Ford 86\n","Name: brand, dtype: int64"]},"metadata":{},"execution_count":8}]},{"cell_type":"markdown","source":["Let's convert the review text into a list of reviews"],"metadata":{"id":"cw1I0la6V_ID"}},{"cell_type":"code","source":["texts = reviews_df['description'].tolist()"],"metadata":{"id":"Iv3ERroL-5d2"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["Let's examine one review in particular"],"metadata":{"id":"ADlpp5-HbX5f"}},{"cell_type":"code","source":["texts[4]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":53},"id":"utam3qIeyzvz","executionInfo":{"status":"ok","timestamp":1684782613617,"user_tz":420,"elapsed":14,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"7cc0cc17-93a3-446f-a921-0cdce0a50cfc"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["\"I finally caved trying this after reading so many great reviews. Unfortunately, for me, it was not meant to be. Despite the brush being great, I had a hard time applying this without getting too clumpy. I didn't find the result to be as volumizing as advertised and it smudged after only a few hours on me. Sigh. Conclusion: this mascara is not so sexy.\""],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":10}]},{"cell_type":"markdown","metadata":{"id":"rTDNuhAqyMQC"},"source":["



\n","\n","## **Encode/tokenize text data for BERT**\n","\n","Next we need to transform our reviews into a format that BERT (via Huggingface) will understand. This is called *encoding* or *tokenizing* the data.\n","\n","We will tokenize the reviews with the `tokenizer()` from HuggingFace's `DistilBertTokenizerFast`. Here's what the `tokenizer()` will do:\n","\n","1. Truncate the texts if they're more than 512 tokens or pad them if they're fewer than 512 tokens. If a word is not in BERT's vocabulary, it will be broken up into smaller \"word pieces,\" demarcated by a `##`.\n","\n","2. Add in special tokens to help BERT:\n"," - [CLS] — Start token of every document\n"," - [SEP] — Separator between each sentence\n"," - [PAD] — Padding at the end of the document as many times as necessary, up to 512 tokens\n"," - ## — Start of a \"word piece\""]},{"cell_type":"markdown","metadata":{"id":"XKgQkdodA-Jz"},"source":["Here we will load `DistilBertTokenizerFast` from HuggingFace library, which will help us transform and encode the texts so they can be used with BERT."]},{"cell_type":"code","metadata":{"id":"cTa04n_-ZI-8"},"source":["from transformers import DistilBertTokenizerFast"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"uDuGq_n4pgZX","colab":{"base_uri":"https://localhost:8080/","height":145,"referenced_widgets":["6ea5836ea6fa4b169afc45e4552e88d3","65dddfc7348f47319111ecfcfe7b55a6","d90615a77d154c9390d4321355b0da55","ad8c39455fdc4c6492cbddbdb968a3ea","e1b980b79c554aef8c433a2c758efe9a","c93ed4057b4f4c07aec4a338106ebdb6","ced2c99134e64da580308c6f1414a413","0f9574c4b5ec43aa8c1101b0d1d10897","554e9ca8a6ad443fa360d9dcf5ba6671","ab53d4da31284b358cf0d36e73ce96f5","8a191d081b6b428b8d7f1f1912c47d09","a66c1ca228f947018b5109d4b90e6742","428e73f4b3684b2d9749efdcc03c399c","82832e67726e4c2e8a61d81e5e81cdbc","d0907d18f5d54e839a33ede7ea2c671c","4b4f3c035f76495baf8df4a3cccc4ca2","0583940dd1d7447f95ca00d0106d35b6","52e17f09dfa2455796031a54e30c3182","77f91290cd9b434ca667a53d6cd5df8f","b3d6c2f5ed73410a81da1c5e8f1837dc","d72b4c04eadc4ae69d4e625080acb934","4cb3c4601b39408388add9a2ccb3f217","a2df16cd32104e09bb024d516197578c","16b949707d1446ec8d2cbbcbb56ad1dc","59788bf989c842d4b58ff02aee66154f","35f3660dbceb477ebb1c242b73573c6b","38ebf0b9f040488a9ce9fb8a1b320386","f933cbeeed3045609e7721adc4bba32c","4edf735ed9ed4fd99f5c6474e025b4a7","7046d1cd0e704d90904af8a278df9aef","0c0f37a4160c41059277b61bc412aa56","db46befff00d4388925702c8e9e8e2d7","30e26faca8764512b7c5ec4d050c63d3","afe754ece46340e1bc66149515dbd22f","2d812091561b4e4b983e2bbc2eab7ea1","a45bfcf8685144c18e6934caeef9e59e","3f345e030ab3489bb3c3008d4bbe423c","415dba2a63244f868451dc4245ad68d2","792a2480c3d5493586c1fb88ce1c333f","e7ed2cf27fe44c9abdf6111d0b716951","dd1ab8f05e5b4168aaac7d92f4c0ed33","8f0bf5b6d5954383b8ad9c5c3fb444ab","913bf28f61b94769a3b14c0f08368535","1d6d270d4da149ab8ebbcdafb2765218"]},"executionInfo":{"status":"ok","timestamp":1684782626758,"user_tz":420,"elapsed":1112,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"48f321c2-2bf0-43cb-c35e-646205da2f5a"},"source":["tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading (…)okenizer_config.json: 0%| | 0.00/28.0 [00:00


\n","\n","## **Load pre-trained BERT model**"]},{"cell_type":"markdown","metadata":{"id":"dGVbuDJ0BXQ2"},"source":["Here we will load a pre-trained BERT model. To speed things up we will use a GPU, but using GPU involves a few extra steps.\n","The command `.to(\"cuda\")` moves data from regular memory to the GPU's memory.\n","\n","\n"]},{"cell_type":"code","metadata":{"id":"U1I4FthqzFjW"},"source":["from transformers import DistilBertModel"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"a7k75REXp7UJ","colab":{"base_uri":"https://localhost:8080/","height":121,"referenced_widgets":["5af49b4dad9a46bda97e746f3745cf97","dc95e652a1f944e88d685c7993c63794","38571b5f77fe462e9fcefea391347d98","0260f68e741b462db70933abee84083b","efb1729e64ad44dda977a52fbdf7f6ec","673de86e2d1a48d484654ccf93a0bb60","051d5a95aee8408ea0059467151e64b9","bd20f60eb60543f0880c74d143e32135","58cdaea6a49e4d099fae61986386a840","7afc0329bc1742e8993596450a1de678","8f0baa6e3eed478d8148acccac0da4e5"]},"executionInfo":{"status":"ok","timestamp":1684782638113,"user_tz":420,"elapsed":4426,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"4b667f5e-4947-4211-8d3c-2dde7c92a595"},"source":["model = DistilBertModel.from_pretrained('distilbert-base-uncased').to(\"cuda\")"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading pytorch_model.bin: 0%| | 0.00/268M [00:00


\n","\n","## **Get BERT word embeddings for each document in a collection**"]},{"cell_type":"markdown","metadata":{"id":"W-vSpKnKBZWJ"},"source":["To get word embeddings for all the words in our collection, we will use a `for` loop.\n","\n","For each review in our list, we will tokenize the review, and we will extract the vocabulary word ID for each word/token in the review (to use for later reference). Then we will run the tokenized review through the BERT model and extract the vectors for each word/token in the review.\n","\n","We thus create two big lists for all the reviews in our collection — `doc_word_ids` and `doc_word_vectors`."]},{"cell_type":"code","metadata":{"id":"j58TvTCd_kxI"},"source":["# List of vocabulary word IDs for all the words in each document (aka each review)\n","doc_word_ids = []\n","\n","# List of word vectors for all the words in each document (aka each review)\n","doc_word_vectors = []\n","\n","# Below we will slice our review to ignore the first (0th) and last (-1) special BERT tokens\n","start_of_words = 1\n","end_of_words = -1\n","\n","# Below we will index the 0th or first document, which will be the only document, since we're analzying one review at a time\n","first_document = 0\n","\n","for i, review in enumerate(texts):\n","\n"," # Here we tokenize each review with the DistilBERT Tokenizer\n"," inputs = tokenizer(review, return_tensors=\"pt\", truncation=True, padding=True)\n","\n"," # Here we extract the vocabulary word ids for all the words in the review (the first or 0th document, since we only have one document)\n"," # We ignore the first and last special BERT tokens\n"," # We also convert from a Pytorch tensor to a numpy array\n"," doc_word_ids.append(inputs.input_ids[first_document].numpy()[start_of_words:end_of_words])\n","\n"," # Here we send the tokenized reviews to the GPU\n"," # The model is already on the GPU, but this review isn't, so we send it to the GPU\n"," inputs.to(\"cuda\")\n"," # Here we run the tokenized review through the DistilBERT model\n"," outputs = model(**inputs)\n","\n"," # We take every element from the first or 0th document, from the 2nd to the 2nd to last position\n"," # Grabbing the last layer is one way of getting token vectors. There are different ways to get vectors with different pros and cons\n"," doc_word_vectors.append(outputs.last_hidden_state[first_document,start_of_words:end_of_words,:].detach().cpu().numpy())\n"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"B4pV1fw61rgT"},"source":["Confirm that we have the same number of documents for both the tokens and the vectors:"]},{"cell_type":"code","metadata":{"id":"RFYsrpeF3C43","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1684782681082,"user_tz":420,"elapsed":32,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"37ffcc43-5ad2-477a-ad5c-e9b667a638c7"},"source":["len(doc_word_ids), len(doc_word_vectors)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(5018, 5018)"]},"metadata":{},"execution_count":18}]},{"cell_type":"code","metadata":{"id":"91SFkQGVHyDV","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1684782681082,"user_tz":420,"elapsed":9,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"23c80278-ae40-4daf-c9f2-90d167d3bdf4"},"source":["doc_word_ids[0], doc_word_vectors[0]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(array([ 2023, 27700, 2003, 9643, 999, 1045, 2245, 2009, 2052,\n"," 2022, 6429, 1998, 2001, 2061, 7568, 2008, 2009, 2001,\n"," 2200, 2152, 3737, 1998, 6450, 2021, 2009, 2001, 2107,\n"," 1037, 10520, 1012, 2028, 5435, 2006, 1998, 2009, 2001,\n"," 2061, 18856, 24237, 2100, 999, 2036, 1010, 2011, 1996,\n"," 2203, 1997, 1996, 2154, 2026, 2104, 2159, 2020, 2304,\n"," 2013, 1996, 27700, 999, 2074, 9202]),\n"," array([[ 0.04701908, 0.07452597, -0.00122089, ..., -0.26615995,\n"," 0.36384752, 0.26436695],\n"," [ 0.94092596, 0.17063166, -0.17726046, ..., -0.22725125,\n"," -0.29596478, -0.34577617],\n"," [-0.03995894, -0.02058499, -0.01363298, ..., -0.13700499,\n"," 0.12584735, 0.62158096],\n"," ...,\n"," [ 0.24695507, 0.30529833, 0.20497079, ..., 0.14400595,\n"," 0.5413362 , 0.00972528],\n"," [ 0.36152834, -0.09225793, 0.2666604 , ..., 0.092596 ,\n"," 0.11792538, 0.9894844 ],\n"," [ 0.531691 , 0.23037525, -0.11810116, ..., 0.08975727,\n"," 0.11264674, -0.5665098 ]], dtype=float32))"]},"metadata":{},"execution_count":19}]},{"cell_type":"markdown","metadata":{"id":"AyVRWw8Icmn-"},"source":["



\n","\n","## **Concatenate all word IDs/vectors for all documents**"]},{"cell_type":"markdown","metadata":{"id":"Gf-9YAo814wM"},"source":["Each element of these lists contains all the tokens/vectors for one document. But we want to concatenate them into two giant collections."]},{"cell_type":"code","metadata":{"id":"afOPAzVoczRW"},"source":["all_word_ids = np.concatenate(doc_word_ids)\n","all_word_vectors = np.concatenate(doc_word_vectors, axis=0)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"wQafcvCqt0Uf"},"source":["We want to make comparisons between vectors quickly. One common option is *cosine similarity*, which measures the angle between vectors but ignores their length. We can speed this computation up by setting all the review vectors to have length 1.0."]},{"cell_type":"code","metadata":{"id":"ogECz92C4T09"},"source":["# Calculating the length of each vector (Pythagorean theorem)\n","row_norms = np.sqrt(np.sum(all_word_vectors ** 2, axis=1))\n","# Dividing every vector by its length\n","all_word_vectors /= row_norms[:,np.newaxis]"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"iBWkQbFLc6kt"},"source":["



\n","\n","## **Find all word positions in a collection**"]},{"cell_type":"markdown","metadata":{"id":"rUB1orgCek1D"},"source":["We can use the array `all_word_ids` to find all the places, or *positions*, in the collection where a word appears.\n","\n","We can find a word's vocab ID in BERT with `tokenizer.vocab` and then check to see where/how many times this ID occurs in `all_word_ids`."]},{"cell_type":"code","metadata":{"id":"bjjVsV8F1UvN"},"source":["def get_word_positions(words):\n","\n"," \"\"\"This function accepts a list of words, rather than a single word\"\"\"\n","\n"," # Get word/vocabulary ID from BERT for each word\n"," word_ids = [tokenizer.vocab[word] for word in words]\n","\n"," # Find all the positions where the words occur in the collection\n"," word_positions = np.where(np.isin(all_word_ids, word_ids))[0]\n","\n"," return word_positions"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"9bB-69p-m4q7"},"source":["Here we'll check to see all the places where the word \"sensitive\" appears in the collection."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"OZKWT3L32GZM","executionInfo":{"status":"ok","timestamp":1684782681413,"user_tz":420,"elapsed":3,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"e3682ca0-1c54-45b0-e501-003512b7c9b9"},"source":["get_word_positions([\"sensitive\"])"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([ 6092, 10738, 14997, 15153, 29942, 41575, 41939, 63591,\n"," 70013, 79906, 83863, 85581, 98966, 100207, 101808, 113760,\n"," 132861, 133107, 139456, 141039, 143913, 150184, 151975, 152983,\n"," 153478, 153664, 155722, 158101, 158191, 158462, 158588, 158733,\n"," 159408, 159555, 160489, 160519, 161026, 164309, 164446, 165695,\n"," 166768, 169200, 169807, 169993, 170953, 173125, 174005, 174135,\n"," 174318, 176044, 177231, 177921, 178412, 178696, 179240, 180165,\n"," 180317, 180691, 181207, 182832, 183028, 183982, 185382, 185398,\n"," 187400, 187941, 188768, 189513, 192046, 198649, 201040, 204296,\n"," 204679, 209885, 218240, 218641, 221080, 222657, 224083, 227826,\n"," 231586, 234207, 237730, 239393, 240547, 262200, 263270, 266501,\n"," 272143, 281776, 281825, 286571, 287928, 287972, 294293, 294672,\n"," 308101, 325566, 325776, 331010, 331186, 333168, 335337, 335643,\n"," 350574, 354482, 354807, 359220, 360683, 360778, 361831, 363309,\n"," 370747, 371587, 371622, 372567])"]},"metadata":{},"execution_count":23}]},{"cell_type":"code","metadata":{"id":"ukKSAiSV3E8a"},"source":["word_positions = get_word_positions([\"sensitive\"])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"kx35pWc0dqIT"},"source":["



\n","\n","## **Find word from word position**"]},{"cell_type":"markdown","metadata":{"id":"9db1FKmAm-7p"},"source":["Nice! Now we know all the positions where the word \"sensitive\" appears in the collection. But it would be more helpful to know the actual words that appear in context around it. To find these context words, we have to convert position IDs back into words."]},{"cell_type":"code","metadata":{"id":"t_0q-9Uodt__"},"source":["# Here we create an array so that we can go backwards from numeric token IDs to words\n","word_lookup = np.empty(tokenizer.vocab_size, dtype=\"O\")\n","\n","for word, index in tokenizer.vocab.items():\n"," word_lookup[index] = word"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"U_Dt6HyVsidn"},"source":["Now we can use `word_lookup` to find a word based on its position in the collection."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4vSMuKJBoLuN","executionInfo":{"status":"ok","timestamp":1684782681578,"user_tz":420,"elapsed":4,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"36ebf7f5-c8e0-48a2-c4c8-7901659e3524"},"source":["word_positions = get_word_positions([\"sensitive\"])\n","\n","for word_position in word_positions:\n"," print(word_position, word_lookup[all_word_ids[word_position]])"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["6092 sensitive\n","10738 sensitive\n","14997 sensitive\n","15153 sensitive\n","29942 sensitive\n","41575 sensitive\n","41939 sensitive\n","63591 sensitive\n","70013 sensitive\n","79906 sensitive\n","83863 sensitive\n","85581 sensitive\n","98966 sensitive\n","100207 sensitive\n","101808 sensitive\n","113760 sensitive\n","132861 sensitive\n","133107 sensitive\n","139456 sensitive\n","141039 sensitive\n","143913 sensitive\n","150184 sensitive\n","151975 sensitive\n","152983 sensitive\n","153478 sensitive\n","153664 sensitive\n","155722 sensitive\n","158101 sensitive\n","158191 sensitive\n","158462 sensitive\n","158588 sensitive\n","158733 sensitive\n","159408 sensitive\n","159555 sensitive\n","160489 sensitive\n","160519 sensitive\n","161026 sensitive\n","164309 sensitive\n","164446 sensitive\n","165695 sensitive\n","166768 sensitive\n","169200 sensitive\n","169807 sensitive\n","169993 sensitive\n","170953 sensitive\n","173125 sensitive\n","174005 sensitive\n","174135 sensitive\n","174318 sensitive\n","176044 sensitive\n","177231 sensitive\n","177921 sensitive\n","178412 sensitive\n","178696 sensitive\n","179240 sensitive\n","180165 sensitive\n","180317 sensitive\n","180691 sensitive\n","181207 sensitive\n","182832 sensitive\n","183028 sensitive\n","183982 sensitive\n","185382 sensitive\n","185398 sensitive\n","187400 sensitive\n","187941 sensitive\n","188768 sensitive\n","189513 sensitive\n","192046 sensitive\n","198649 sensitive\n","201040 sensitive\n","204296 sensitive\n","204679 sensitive\n","209885 sensitive\n","218240 sensitive\n","218641 sensitive\n","221080 sensitive\n","222657 sensitive\n","224083 sensitive\n","227826 sensitive\n","231586 sensitive\n","234207 sensitive\n","237730 sensitive\n","239393 sensitive\n","240547 sensitive\n","262200 sensitive\n","263270 sensitive\n","266501 sensitive\n","272143 sensitive\n","281776 sensitive\n","281825 sensitive\n","286571 sensitive\n","287928 sensitive\n","287972 sensitive\n","294293 sensitive\n","294672 sensitive\n","308101 sensitive\n","325566 sensitive\n","325776 sensitive\n","331010 sensitive\n","331186 sensitive\n","333168 sensitive\n","335337 sensitive\n","335643 sensitive\n","350574 sensitive\n","354482 sensitive\n","354807 sensitive\n","359220 sensitive\n","360683 sensitive\n","360778 sensitive\n","361831 sensitive\n","363309 sensitive\n","370747 sensitive\n","371587 sensitive\n","371622 sensitive\n","372567 sensitive\n"]}]},{"cell_type":"markdown","metadata":{"id":"0KwzuaBxpkwG"},"source":["We can also look for the 3 words that come before \"sensitive\" and the 3 words that come after it."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"eJ_zg3cNfJum","executionInfo":{"status":"ok","timestamp":1684782762313,"user_tz":420,"elapsed":184,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"9483a876-b0ab-46c4-c378-32d4f5f93be8"},"source":["word_positions = get_word_positions([\"sensitive\"])\n","\n","for word_position in word_positions:\n","\n"," # Slice 3 words before \"user\"\n"," start_pos = word_position - 3\n"," # Slice 3 words after \"user\"\n"," end_pos = word_position + 4\n","\n"," context_words = word_lookup[all_word_ids[start_pos:end_pos]]\n"," # Join the words together\n"," context_words = ' '.join(context_words)\n"," print(word_position, context_words)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["6092 not for the sensitive type this is\n","10738 good for my sensitive eyes and for\n","14997 mascara ##s or sensitive eyes i would\n","15153 i have very sensitive eyes and most\n","29942 ##rita ##te my sensitive eyes , and\n","41575 really watery and sensitive eyes , so\n","41939 nice for my sensitive eyes ! …\n","63591 great for my sensitive , watery eyes\n","70013 i have very sensitive dry eyes and\n","79906 eyes are so sensitive . it does\n","83863 ##proof mascara for sensitive eyes . this\n","85581 surprisingly great for sensitive eyes , and\n","98966 you have super sensitive eyes with contact\n","100207 i have really sensitive eyes , and\n","101808 rub on your sensitive eye area .\n","113760 ' m super sensitive when it comes\n","132861 i have very sensitive eyes , rosa\n","133107 aren ' t sensitive so i got\n","139456 . i have sensitive eyes so eye\n","141039 eyes can be sensitive to mascara .\n","143913 someone with super sensitive eyes and i\n","150184 due to my sensitive eyes . application\n","151975 ##er ##gies or sensitive eyes because your\n","152983 ##y , super sensitive skin and really\n","153478 not for my sensitive skin . i\n","153664 have combination , sensitive skin and i\n","155722 not for my sensitive skin . this\n","158101 if you are sensitive to strong scents\n","158191 have extremely dry sensitive ec ##ze ##ma\n","158462 ##ne prone / sensitive skin ! i\n","158588 that is rather sensitive . i am\n","158733 out my very sensitive skin and i\n","159408 i have very sensitive ac ##ne prone\n","159555 have tried their sensitive skin por ##eles\n","160489 on my sometimes sensitive skin . i\n","160519 t really have sensitive skin and it\n","161026 t make my sensitive skin break out\n","164309 i suffer from sensitive combination skin that\n","164446 has gotten quite sensitive , and with\n","165695 irritation to my sensitive skin . i\n","166768 crank ##y / sensitive skin breakout !\n","169200 . i have sensitive , ac ##ne\n","169807 my skin is sensitive and reactive ,\n","169993 i have combination sensitive skin so it\n","170953 . i have sensitive skin that literally\n","173125 my skin is sensitive and dry ,\n","174005 . i have sensitive oil ##y skin\n","174135 i also have sensitive skin and don\n","174318 if you have sensitive dry skin .\n","176044 soothe ##s my sensitive skin . unlike\n","177231 for my combination sensitive skin . from\n","177921 skin is super sensitive to silicon ##es\n","178412 amazing ! having sensitive , highly allergic\n","178696 ( i have sensitive skin ) ,\n","179240 ##ne prone and sensitive to products .\n","180165 nice formula for sensitive skin , it\n","180317 , or have sensitive skin ! …\n","180691 i have super sensitive skin and just\n","181207 have dry , sensitive skin . this\n","182832 . i have sensitive skin too and\n","183028 i have very sensitive fair skin .\n","183982 since i have sensitive skin , i\n","185382 i was so sensitive . could not\n","185398 for people with sensitive eyes . i\n","187400 do not have sensitive eyes in general\n","187941 i have pretty sensitive eyes , and\n","188768 work with my sensitive eyes and oil\n","189513 ##d and too sensitive eyes … read\n","192046 someone with very sensitive eyes . i\n","198649 eyes can be sensitive but this liner\n","201040 i have extremely sensitive , all ##ergy\n","204296 to those with sensitive eyes ! i\n","204679 i also have sensitive all ##ergy prone\n","209885 : i have sensitive eyes ( all\n","218240 someone who has sensitive skin or eyes\n","218641 ##rita ##te my sensitive eyes . my\n","221080 do not have sensitive eyes , or\n","222657 i have extremely sensitive eyes and wear\n","224083 and nice for sensitive eyes wow ,\n","227826 eyes which are sensitive . as eye\n","231586 this formula is sensitive to room temperature\n","234207 forever ' s sensitive eye remove ##r\n","237730 good for my sensitive skin : i\n","239393 i have really sensitive eyes and this\n","240547 more i have sensitive easily irritated eyes\n","262200 more i have sensitive eyes and with\n","263270 ' ve got sensitive eyes and oil\n","266501 comfortable on my sensitive eyes than the\n","272143 your eyes are sensitive and tend to\n","281776 can dry out sensitive eyelids . the\n","281825 not bother my sensitive skin . as\n","286571 if you have sensitive skin , i\n","287928 harsh for the sensitive eye ##lid skin\n","287972 my generally moderately sensitive skin . i\n","294293 i have super sensitive eyelids and have\n","294672 but i have sensitive skin and it\n","308101 entire body is sensitive , so i\n","325566 t bother my sensitive eyes ! !\n","325776 i have extremely sensitive eyes and it\n","331010 i have very sensitive eyes , and\n","331186 eyes are very sensitive and this eye\n","333168 eyes are really sensitive and i my\n","335337 ##rita ##te my sensitive eyes . it\n","335643 . i have sensitive eyes and this\n","350574 i have really sensitive skin and it\n","354482 ! i have sensitive skin and have\n","354807 s good for sensitive skin ! i\n","359220 become very heat sensitive . even in\n","360683 ' s very sensitive & this product\n","360778 i have extremely sensitive skin that is\n","361831 ##rita ##te my sensitive skin , and\n","363309 sadly . my sensitive skin doesn '\n","370747 have combination , sensitive skin and live\n","371587 ##ne prone / sensitive skin . i\n","371622 may help other sensitive skin girls avoid\n","372567 asian , super sensitive , oil ##y\n"]}]},{"cell_type":"markdown","metadata":{"id":"_9ssdFSBANkN"},"source":["Let's make some functions that will help us get the context words around a certain word position for whatever size window (certain number of words before and after) that we want.\n","\n","The first function `get_context()` will simply return the tokens without cleaning them, and the second function `get_context_clean()` will return the tokens in a more readable fashion."]},{"cell_type":"code","metadata":{"id":"DQDmbTABgQF9"},"source":["def get_context(word_id, window_size=10):\n","\n"," \"\"\"Simply get the tokens that occur before and after word position\"\"\"\n","\n"," start_pos = max(0, word_id - window_size) # The token where we will start the context view\n"," end_pos = min(word_id + window_size + 1, len(all_word_ids)) # The token where we will end the context view\n","\n"," # Make a list called tokens and use word_lookup to get the words for given token IDs from starting position up to the keyword\n"," tokens = [word_lookup[word] for word in all_word_ids[start_pos:end_pos] ]\n","\n"," context_words = \" \".join(tokens)\n","\n"," return context_words"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"5lHshYA2-zxY"},"source":["import re\n","\n","def get_context_clean(word_id, window_size=10):\n","\n"," \"\"\"Get the tokens that occur before and after word position AND make them more readable\"\"\"\n","\n"," keyword = word_lookup[all_word_ids[word_id]]\n"," start_pos = max(0, word_id - window_size) # The token where we will start the context view\n"," end_pos = min(word_id + window_size + 1, len(all_word_ids)) # The token where we will end the context view\n","\n"," # Make a list called tokens and use word_lookup to get the words for given token IDs from starting position up to the keyword\n"," tokens = [word_lookup[word] for word in all_word_ids[start_pos:end_pos] ]\n","\n"," # Make wordpieces slightly more readable\n"," # This is probably not the most efficient way to clean and correct for weird spacing\n"," context_words = \" \".join(tokens)\n"," context_words = re.sub(r'\\s+([##])', r'\\1', context_words)\n"," context_words = re.sub(r'##', r'', context_words)\n"," context_words = re.sub('\\s+\\'s', '\\'s', context_words)\n"," context_words = re.sub('\\s+\\'d', '\\'d', context_words)\n"," context_words = re.sub('\\s\\'er', '\\'er', context_words)\n"," context_words = re.sub(r'\\s+([-,:?.!;])', r'\\1', context_words)\n"," context_words = re.sub(r'([-\\'\"])\\s+', r'\\1', context_words)\n"," context_words = re.sub('\\s+\\'s', '\\'s', context_words)\n"," context_words = re.sub('\\s+\\'d', '\\'d', context_words)\n","\n"," # Bold the keyword by putting asterisks around it\n"," if keyword in context_words:\n"," context_words = re.sub(f\"\\\\b{keyword}\\\\b\", f\"**{keyword}**\", context_words)\n"," context_words = re.sub(f\"\\\\b({keyword}[esdtrlying]+)\\\\b\", fr\"**\\1**\", context_words)\n","\n"," return context_words"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"0-SQVh1zyblT"},"source":["To visualize the search keyword even more easily, we're going to import a couple of Python modules that will allow us to output text with bolded words and other styling. Here we will make a function `print_md()` that will allow us to print with Markdown styling."]},{"cell_type":"code","metadata":{"id":"SPIho4qIGGqp"},"source":["from IPython.display import Markdown, display\n","\n","def print_md(string):\n"," display(Markdown(string))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"UksbSqkagabE","executionInfo":{"status":"ok","timestamp":1684782765019,"user_tz":420,"elapsed":326,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"2c58b18c-422d-4134-94c1-d2126c43f717"},"source":["word_positions = get_word_positions([\"sensitive\"])\n","\n","for word_position in word_positions:\n","\n"," print_md(f\"
{word_position}: {get_context_clean(word_position)}
\")"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
6092: a compliment. worth the money. not for the **sensitive** type this is not at all worth the hype
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
10738: the day. this product was not good for my **sensitive** eyes and for the price i think it's
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
14997: have problems with the staying power of mascaras or **sensitive** eyes i would skip this one. … read more
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
15153: brand. i absolutely loved it. i have very **sensitive** eyes and most mascaras irritate my eyes
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
29942: udge easy, doesn 't irritate my **sensitive** eyes, and gives my amazing definition and volume!
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
41575: ! to start off, i have really watery and **sensitive** eyes, so i can only wear waterproof mascara
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
41939: to, so less rubbing is really nice for my **sensitive** eyes! … read more this mascara is absolutely amazing
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
63591: . … read more this mascara is great for my **sensitive**, watery eyes and oily lids. stays put
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
70013: also keeps my lashes very soft. i have very **sensitive** dry eyes and this does not aggravate them
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
79906: it was a fiber mascara since my eyes are so **sensitive**. it does work and doesn 't smudge
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
83863: was told this was the best waterproof mascara for **sensitive** eyes. this stuff feels like razors when it
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
85581: fab i decided this mascara is surprisingly great for **sensitive** eyes, and for people who are looking for a
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
98966: starts to get dried. so if you have super **sensitive** eyes with contact lenses, you'd better not
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
100207: to take it off at night. i have really **sensitive** eyes, and my eyes don 't burn from
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
101808: you dont have to tug & rub on your **sensitive** eye area. i tried it with the dior
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
113760: t seem dry to me. i 'm super **sensitive** when it comes to which products work for me,
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
132861: ation i read on a forum. i have very **sensitive** eyes, rosacea. 99 % of mascaras
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
133107: is a bit strong but my eyes aren 't **sensitive** so i got use to the smell. the smell
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
139456: this product in exchange for a review. i have **sensitive** eyes so eye cream shadow etc can cause tears,
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
141039: and last all day long! my eyes can be **sensitive** to mascara. any flaking and i am it
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
143913: this came out. i 'm someone with super **sensitive** eyes and i was hesitant at first to try it
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
150184: my makeup remover which is important due to my **sensitive** eyes. application is super easy especially with the shape
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
151975: goodluck if you have allergies or **sensitive** eyes because your eyes will water and the liner comes
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
152983: this primer. i have oily, super **sensitive** skin and really dislike silicone-based primers
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
153478: , but i guess it ’ s not for my **sensitive** skin. i have been in the skin care business
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
153664: !!!! wow so i have combination, **sensitive** skin and i'd been hearing a lot about
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
155722: holding my makeup in place, but not for my **sensitive** skin. this didn 't work well with my
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
158101: in areas. it smells great. if you are **sensitive** to strong scents then you may want to avoid it
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
158191: … read more great product! i have extremely dry **sensitive** eczema skin and this primer helps soothe
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
158462: my combo / oily / acne prone / **sensitive** skin! i really like how my make up looks
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
158588: objectives. i have very dry skin that is rather **sensitive**. i am not particularly acne prone, but
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
158733: on my face, but ultimately broke out my very **sensitive** skin and i had to stop using it. which
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
159408: reviews about it breaking people out, i have very **sensitive** acne prone skin and my skin has actually improved
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
159555: or is it just me? i have tried their **sensitive** skin poreless primer and was not happy
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
160489: for me as well as very gentle on my sometimes **sensitive** skin. i now use this daily and love it
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
160519: work for me, i don 't really have **sensitive** skin and it made me breakout in places that are
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
161026: it smells like coconut and doesn 't make my **sensitive** skin break out like some primers do. absolutely
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
164309: lovee love loveee this! i suffer from **sensitive** combination skin that is severely dry during the winter months
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
164446: the couple of years, my skin has gotten quite **sensitive**, and with that, quite dehydrated
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
165695: doesn 't cause redness or irritation to my **sensitive** skin. i love the pump dispenser
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
166768: t irritate or make my cranky / **sensitive** skin breakout! no silicones! works fantastic,
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
169200: and really gives my makeup staying power. i have **sensitive**, acne-prone skin and have had no
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
169807: it's a beautiful product. my skin is **sensitive** and reactive, no issues. in fact, i
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
169993: primer i have ever used. i have combination **sensitive** skin so it can get dry randomly. other prime
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
170953: all while making my skin feel smooth. i have **sensitive** skin that literally flares up if i lightly rub
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
173125: super excited to try this product. my skin is **sensitive** and dry, so i thought coconut water sounded great
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
174005: all day even in 85 degree weather. i have **sensitive** oily skin and this is perfect for me.
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
174135: y-i love this product-i also have **sensitive** skin and don 't like to wear too much
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
174318: !! i would deff recommended if you have **sensitive** dry skin. it works wonders. i love how
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
176044: lotion, which i love and soothes my **sensitive** skin. unlike with most of the silicone based
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
177231: . i have tried every primer for my combination **sensitive** skin. from ysl, to laura mercier
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
177921: primers at all costs because my skin is super **sensitive** to silicones. i also have combination oil-
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
178412: dressed up and it's just amazing! having **sensitive**, highly allergic skin and rosacea, i was
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
178696: how it made some people break out ( i have **sensitive** skin ), but i haven 't had any
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
179240: my skin is oily and acne prone and **sensitive** to products. this goes on smooth like a lightweight
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
180165: caking. it also is very nice formula for **sensitive** skin, it doesn 't have a strong scent
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
180317: acne-prone, oily, or have **sensitive** skin! … read more best primer for dry
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
180691: . plus the scent is lovely. i have super **sensitive** skin and just about any type of lotion makes
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
181207: makes my skin look great! i have dry, **sensitive** skin. this means i have to wear a moist
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
182832: sunscreen. love this primer. i have **sensitive** skin too and no problems with this. light coconut
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
183028: product is definitely not good for me i have very **sensitive** fair skin. i foolishly didn 't read
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
183982: one, it peaked my interest. since i have **sensitive** skin, i always read a number of reviews for
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
185382: waterline let alone lash line because i was so **sensitive**. could not keep my eyes open and would not
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
185398: eyes open and would not recommend it for people with **sensitive** eyes. i have been trying out this pencil for
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
187400: lining my waterline for years and do not have **sensitive** eyes in general but for some reason i can '
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
187941: t have any smudging. i have pretty **sensitive** eyes, and this didn 't bother them one
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
188768: the hunt for something that 'll work with my **sensitive** eyes and oil-slick eyelids. i bought this
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
189513: . maybe i just have a dud and too **sensitive** eyes … read more this is probably the 10th eye
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
192046: ... which is coming from someone with very **sensitive** eyes. i also use this to tightline with
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
198649: fan of waterproof makeup since my eyes can be **sensitive** but this liner does not irritate my eyes
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
201040: let me first preface by stating that i have extremely **sensitive**, allergy-prone, and watery eyes.
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
204296: 's fine. just a warning to those with **sensitive** eyes! i love the vibrance of the
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
204679: the eyeliner on very difficult. i also have **sensitive** allergy prone eyes which is why i don '
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
209885: what to do? a few notes: i have **sensitive** eyes ( allergies ) that water terribly when
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
218240: recommend this for every day use to someone who has **sensitive** skin or eyes. recently received this as a birthday
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
218641: only eyeliner that does not irritate my **sensitive** eyes. my new favorite! i bought the high
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
221080: only recommend this product to those who do not have **sensitive** eyes, or do not react to use of this
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
222657: on smooth, quick and creamy. i have extremely **sensitive** eyes and wear glasses, and a lot of liner
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
224083: . the very best. easy application and nice for **sensitive** eyes wow, i love this eyeliner! i
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
227826: and does not irritate my eyes which are **sensitive**. as eye liner goes, it's about
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
231586: ). i am not sure if this formula is **sensitive** to room temperature, but it was warm in the
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
234207: -tip ( i recommend make up forever's **sensitive** eye remover. it's blue and so
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
237730: as well. as far as being good for my **sensitive** skin: i accidentally stabbed myself in the eye twice
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
239393: black, last all day.. i have really **sensitive** eyes and this does not irratate at all
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
240547: seen about this liner. … read more i have **sensitive** easily irritated eyes and this does not bother them at
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
262200: gets rid of it! … read more i have **sensitive** eyes and with all other liquid eyeliners they
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
263270: eyeliner of all time. i 've got **sensitive** eyes and oily eyelids, so i 've
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
266501: 's brush tip is much more comfortable on my **sensitive** eyes than the felt tip on kat von d '
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
272143: first; if, for example, your eyes are **sensitive** and tend to tear up when you 're applying
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
281776: ruin such a good thing? it can dry out **sensitive** eyelids. the consistency is pretty thick and you only
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
281825: 's smooth and creamy and does not bother my **sensitive** skin. as for how well it works i have
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
286571: to heal for several days after. if you have **sensitive** skin, i would not recommend this product to you
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
287928: me. however, it is really harsh for the **sensitive** eyelid skin. i had no troubles at first
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
287972: up but it is too harsh for my generally moderately **sensitive** skin. i 'm 32 so definitely at the
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
294293: -aging version is much better. i have super **sensitive** eyelids and have reacted badly to other brands of prime
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
294672: sure if sephora carries it yet but i have **sensitive** skin and it literally melts makeup away gently and
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
308101: already sampling at the store. my entire body is **sensitive**, so i had to be real careful & pic
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
325566: this pen, and it doesn 't bother my **sensitive** eyes!!! will repurchase this great
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
325776: amazing for many reasons: 1. i have extremely **sensitive** eyes and it doesn 't even tingle when i
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
331010: 't be disappointed! also-i have very **sensitive** eyes, and this product has not caused any problems
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
331186: hypoallergenic, my eyes are very **sensitive** and this eyeliner does not irritate them
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
333168: all kinds of eyeliner. my eyes are really **sensitive** and i my eyes randomly start crying through out the
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
335337: plus, it doesn 't irritate my **sensitive** eyes. it's worth every penny. as
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
335643: waterproof eyeliner that stays put. i have **sensitive** eyes and this does not burn. no racco
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
350574: best products ive ever bought! i have really **sensitive** skin and it breaks out or gets irritated pretty easily
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
354482: but it ’ s a great product! i have **sensitive** skin and have always struggled to find the right products
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
354807: make me break out so it's good for **sensitive** skin! i 'll be buying this again!
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
359220: . since then, my body has become very heat **sensitive**. even in the dead of winter, i experience
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
360683: i have normal to dry skin that's very **sensitive** & this product does not dry out or irrita
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
360778: at the end of the shift. i have extremely **sensitive** skin that is acne prone, and this has
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
361831: -greasy, didn 't irritate my **sensitive** skin, and i didn 't notice any fragrance
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
363309: skin itchy. i returned it sadly. my **sensitive** skin doesn 't seem to care much for it
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
370747: t do anything for me. i have combination, **sensitive** skin and live in a pretty humid environment ( houston
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
371587: still work for girls with less acne prone / **sensitive** skin. i submitted a similar review of this product
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
371622: hopefully this review will, since it may help other **sensitive** skin girls avoid a major breakout. … read more
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":[""],"text/markdown":"
372567: it everyday!! i 'm asian, super **sensitive**, oily, acne prone skin and i
"},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"FHpPAIpHelDv"},"source":["Here we make a list of all the context views for our keyword."]},{"cell_type":"code","metadata":{"id":"jBKdEKzO3ndD"},"source":["word_positions = get_word_positions([\"sensitive\"])\n","\n","keyword_contexts = []\n","keyword_contexts_tokens = []\n","\n","for position in word_positions:\n","\n"," keyword_contexts.append(get_context_clean(position))\n"," keyword_contexts_tokens.append(get_context(position))"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"FOW3UJCU941G"},"source":["



\n","\n","## **Get word vectors and reduce them with PCA**"]},{"cell_type":"markdown","metadata":{"id":"Xggu6BRkAgGu"},"source":["Finally, we don't just want to *read* all the instances of \"sensitive\" in the collection, we want to *measure* the similarity of all the instances of \"sensitive.\"\n","\n","To measure similarity between all the instances of \"sensitive,\" we will take the vectors for each instance and then use PCA to reduce each 768-dimensionsal vector to the 2 dimensions that capture the most variation."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":75},"id":"ZnaABMiLsTAQ","executionInfo":{"status":"ok","timestamp":1684782769250,"user_tz":420,"elapsed":187,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"a6c52fe3-4f0c-4a30-993f-6263213a61b3"},"source":["from sklearn.decomposition import PCA\n","\n","word_positions = get_word_positions([\"sensitive\"])\n","\n","pca = PCA(n_components=2)\n","\n","pca.fit(all_word_vectors[word_positions,:].T)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["PCA(n_components=2)"],"text/html":["
PCA(n_components=2)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
"]},"metadata":{},"execution_count":33}]},{"cell_type":"markdown","metadata":{"id":"nPWjXDxpevTe"},"source":["Then, for convenience, we will put these PCA results into a Pandas DataFrame, which will use to generate an interactive plot."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"XUxuru27DrMm","executionInfo":{"status":"ok","timestamp":1684782771586,"user_tz":420,"elapsed":6,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"466ea714-8f0b-45de-bea0-7d716e64d7d6"},"source":["df_to_plot = pd.DataFrame({\"x\": pca.components_[0,:],\n"," \"y\": pca.components_[1,:],\n"," \"context\": keyword_contexts,\n"," \"tokens\": keyword_contexts_tokens})\n","df_to_plot.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":[" x y \\\n","0 -0.079377 0.017588 \n","1 -0.095697 -0.080537 \n","2 -0.095200 -0.087407 \n","3 -0.094516 -0.149339 \n","4 -0.093228 -0.060299 \n","\n"," context \\\n","0 a compliment. worth the money. not for the **sensitive** type this is not at all worth the hype \n","1 the day. this product was not good for my **sensitive** eyes and for the price i think it's \n","2 have problems with the staying power of mascaras or **sensitive** eyes i would skip this one. … read more \n","3 brand. i absolutely loved it. i have very **sensitive** eyes and most mascaras irritate my eyes \n","4 udge easy, doesn 't irritate my **sensitive** eyes, and gives my amazing definition and volume! \n","\n"," tokens \n","0 a compliment . worth the money . not for the sensitive type this is not at all worth the h ##ype \n","1 the day . this product was not good for my sensitive eyes and for the price i think it ' s \n","2 have problems with the staying power of mascara ##s or sensitive eyes i would skip this one . … read more \n","3 brand . i absolutely loved it . i have very sensitive eyes and most mascara ##s ir ##rita ##te my eyes \n","4 ##udge easy , doesn ' t ir ##rita ##te my sensitive eyes , and gives my amazing definition and volume ! "],"text/html":["\n","
\n","
\n","
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
xycontexttokens
0-0.0793770.017588a compliment. worth the money. not for the **sensitive** type this is not at all worth the hypea compliment . worth the money . not for the sensitive type this is not at all worth the h ##ype
1-0.095697-0.080537the day. this product was not good for my **sensitive** eyes and for the price i think it'sthe day . this product was not good for my sensitive eyes and for the price i think it ' s
2-0.095200-0.087407have problems with the staying power of mascaras or **sensitive** eyes i would skip this one. … read morehave problems with the staying power of mascara ##s or sensitive eyes i would skip this one . … read more
3-0.094516-0.149339brand. i absolutely loved it. i have very **sensitive** eyes and most mascaras irritate my eyesbrand . i absolutely loved it . i have very sensitive eyes and most mascara ##s ir ##rita ##te my eyes
4-0.093228-0.060299udge easy, doesn 't irritate my **sensitive** eyes, and gives my amazing definition and volume!##udge easy , doesn ' t ir ##rita ##te my sensitive eyes , and gives my amazing definition and volume !
\n","
\n"," \n"," \n"," \n","\n"," \n","
\n","
\n"," "]},"metadata":{},"execution_count":34}]},{"cell_type":"markdown","metadata":{"id":"quwSCsFDUywu"},"source":["



\n","\n","## **Match context with original text and metadata**"]},{"cell_type":"markdown","metadata":{"id":"IA3Cd2pgwDcY"},"source":["It's helpful (and fun!) to know where each instance of a word actually comes from. The easiest method we've found for matching a bit of context with its original review and metdata is to 1) add a tokenized version of each review to our original Pandas Dataframe 2) check to see if the context shows up in a review 3) and if so, grab the original review and metadata."]},{"cell_type":"code","metadata":{"id":"DmY6MMGIhWGJ"},"source":["# Tokenize all the reviews\n","tokenized_reviews = tokenizer(texts, truncation=True, padding=True, return_tensors=\"pt\")\n","\n","# Get a list of all the tokens for each review\n","all_tokenized_reviews = []\n","for i in range(len(tokenized_reviews['input_ids'])):\n"," all_tokenized_reviews.append(' '.join(tokenized_reviews[i].tokens))\n","\n","# Add them to the original DataFrame\n","reviews_df['tokens'] = all_tokenized_reviews"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"8Uoqk2yQjQhH"},"source":["def find_original_review(rows):\n","\n"," \"\"\"This function checks to see whether the context tokens show up in the original review,\n"," and if so, returns metadata about the title, author, period, and URL for that review\"\"\"\n","\n"," text = rows['tokens'].replace('**', '')\n"," text = text[55:70]\n","\n"," if reviews_df['tokens'].str.contains(text, regex=False).any() == True :\n"," row = reviews_df[reviews_df['tokens'].str.contains(text, regex=False)].values[0]\n"," stars, brand, name, makeup_type = row[1], row[5], row[6], row[7]\n"," return stars, brand, name, makeup_type\n"," else:\n"," return None, None, None, None"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"KA3qC3py0uwf"},"source":["df_to_plot[['stars', 'brand', 'name', 'type']] = df_to_plot.apply(find_original_review, axis='columns', result_type='expand')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"eu4SZ7R7U8Na"},"source":["



\n","\n","## **Plot word embeddings**"]},{"cell_type":"markdown","metadata":{"id":"g5PQYxa_ezps"},"source":["Lastly, we will plot the words vectors from this DataFrame with the Python data viz library [Altair](https://altair-viz.github.io/gallery/scatter_tooltips.html)."]},{"cell_type":"code","metadata":{"id":"uwyogpt7zkOk"},"source":["import altair as alt"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":585},"id":"hE_kCW9QUDKX","executionInfo":{"status":"ok","timestamp":1684782789594,"user_tz":420,"elapsed":131,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"7930a167-749a-4ba7-b107-5beb886c30a6"},"source":["alt.Chart(df_to_plot, title=\"Word Similarity: Sensitive\").mark_circle(size=200).encode(\n"," alt.X('x',\n"," scale=alt.Scale(zero=False)\n"," ), y=\"y\",\n"," # If you click a point, take you to the URL link\n"," # href=\"link\",\n"," # The categories that show up in the hover tooltip\n"," tooltip=['context', 'type', 'brand', 'name'],\n"," ).interactive().properties(\n"," width=500,\n"," height=500\n",")"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["\n","
\n",""],"text/plain":["alt.Chart(...)"]},"metadata":{},"execution_count":40}]},{"cell_type":"markdown","metadata":{"id":"vLSjJc8-IZJ_"},"source":["



\n","\n","## **Plot word embeddings from keywords (all at once!)**"]},{"cell_type":"markdown","metadata":{"id":"dPXChKeD-Gd6"},"source":["We can put the code from the previous few sections into a single cell and plot the BERT word embeddings for any list of words."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":585},"id":"dA-SdhjlprIj","executionInfo":{"status":"ok","timestamp":1654525438867,"user_tz":240,"elapsed":1893,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"712ca300-1323-497d-eaeb-bbb90ed35547"},"source":["# List of keywords that you want to compare\n","keywords = ['pool', 'gym', 'wedding', 'funeral', 'office', 'party']\n","\n","# How to color the points in the plot. The other option is \"period\" for time period\n","color_by = 'word'\n","\n","# Get all word positions\n","word_positions = get_word_positions(keywords)\n","\n","# Get all contexts around the words\n","keyword_contexts = []\n","keyword_contexts_tokens = []\n","words = []\n","\n","for position in word_positions:\n"," words.append(word_lookup[all_word_ids[position]])\n"," keyword_contexts.append(get_context_clean(position))\n"," keyword_contexts_tokens.append(get_context(position))\n","\n","# Reduce word vectors with PCA\n","pca = PCA(n_components=2)\n","pca.fit(all_word_vectors[word_positions,:].T)\n","\n","# Make a DataFrame with PCA results\n","df_to_plot = pd.DataFrame({\"x\": pca.components_[0,:],\n"," \"y\": pca.components_[1,:],\n"," \"context\": keyword_contexts,\n"," \"tokens\": keyword_contexts_tokens,\n"," \"word\": words})\n","# Match original text and metadata\n","df_to_plot[['stars', 'brand', 'name', 'type']] = df_to_plot.apply(find_original_review, axis='columns', result_type='expand')\n","\n","# Rename columns so that the context shows up as the \"title\" in the tooltip (bigger and bolded)\n","# df = df.rename(columns={'brand': 'brand', 'name': 'title'})\n","\n","# Make the plot\n","alt.Chart(df_to_plot, title=f\"Word Similarity: {', '.join(keywords).title()}\").mark_circle(size=200).encode(\n"," alt.X('x',\n"," scale=alt.Scale(zero=False)\n"," ), y=\"y\",\n"," color= color_by,\n"," # href=\"link\",\n"," tooltip=['word', 'context', 'type', 'brand', 'name', 'stars']\n"," ).interactive().properties(\n"," width=500,\n"," height=500\n",")"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["\n","
\n",""],"text/plain":["alt.Chart(...)"]},"metadata":{},"execution_count":45}]},{"cell_type":"code","source":["# List of keywords that you want to compare\n","keywords = ['smooth', 'sharp', 'clean']\n","\n","# How to color the points in the plot. The other option is \"period\" for time period\n","color_by = 'word'\n","\n","# Get all word positions\n","word_positions = get_word_positions(keywords)\n","\n","# Get all contexts around the words\n","keyword_contexts = []\n","keyword_contexts_tokens = []\n","words = []\n","\n","for position in word_positions:\n"," words.append(word_lookup[all_word_ids[position]])\n"," keyword_contexts.append(get_context_clean(position))\n"," keyword_contexts_tokens.append(get_context(position))\n","\n","# Reduce word vectors with PCA\n","pca = PCA(n_components=2)\n","pca.fit(all_word_vectors[word_positions,:].T)\n","\n","# Make a DataFrame with PCA results\n","df_to_plot = pd.DataFrame({\"x\": pca.components_[0,:],\n"," \"y\": pca.components_[1,:],\n"," \"context\": keyword_contexts,\n"," \"tokens\": keyword_contexts_tokens,\n"," \"word\": words})\n","# Match original text and metadata\n","df_to_plot[['stars', 'brand', 'name', 'type']] = df_to_plot.apply(find_original_review, axis='columns', result_type='expand')\n","\n","# Rename columns so that the context shows up as the \"title\" in the tooltip (bigger and bolded)\n","# df = df.rename(columns={'brand': 'brand', 'name': 'title'})\n","\n","# Make the plot\n","alt.Chart(df_to_plot, title=f\"Word Similarity: {', '.join(keywords).title()}\").mark_circle(size=200).encode(\n"," alt.X('x',\n"," scale=alt.Scale(zero=False)\n"," ), y=\"y\",\n"," color= color_by,\n"," # href=\"link\",\n"," tooltip=['word', 'context', 'type', 'brand', 'name', 'stars']\n"," ).interactive().properties(\n"," width=500,\n"," height=500\n",")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":585},"id":"Ng56lUBpdV70","executionInfo":{"status":"ok","timestamp":1654525442634,"user_tz":240,"elapsed":3769,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"outputId":"db0e5f1a-543b-478d-d9d8-293328d49d8d"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["\n","
\n",""],"text/plain":["alt.Chart(...)"]},"metadata":{},"execution_count":46}]},{"cell_type":"code","source":["# List of keywords that you want to compare\n","keywords = ['long']\n","\n","# How to color the points in the plot. The other option is \"period\" for time period\n","color_by = 'word'\n","\n","# Get all word positions\n","word_positions = get_word_positions(keywords)\n","\n","# Get all contexts around the words\n","keyword_contexts = []\n","keyword_contexts_tokens = []\n","words = []\n","\n","for position in word_positions:\n"," words.append(word_lookup[all_word_ids[position]])\n"," keyword_contexts.append(get_context_clean(position))\n"," keyword_contexts_tokens.append(get_context(position))\n","\n","# Reduce word vectors with PCA\n","pca = PCA(n_components=2)\n","pca.fit(all_word_vectors[word_positions,:].T)\n","\n","# Make a DataFrame with PCA results\n","df_to_plot = pd.DataFrame({\"x\": pca.components_[0,:],\n"," \"y\": pca.components_[1,:],\n"," \"context\": keyword_contexts,\n"," \"tokens\": keyword_contexts_tokens,\n"," \"word\": words})\n","# Match original text and metadata\n","df_to_plot[['stars', 'brand', 'name', 'type']] = df_to_plot.apply(find_original_review, axis='columns', result_type='expand')\n","\n","# Rename columns so that the context shows up as the \"title\" in the tooltip (bigger and bolded)\n","# df = df.rename(columns={'brand': 'brand', 'name': 'title'})\n","\n","# Make the plot\n","alt.Chart(df_to_plot, title=f\"Word Similarity: {', '.join(keywords).title()}\").mark_circle(size=200).encode(\n"," alt.X('x',\n"," scale=alt.Scale(zero=False)\n"," ), y=\"y\",\n"," color= color_by,\n"," # href=\"link\",\n"," tooltip=['word', 'context', 'type', 'brand', 'name', 'stars']\n"," ).interactive().properties(\n"," width=500,\n"," height=500\n",")"],"metadata":{"id":"Zf9mvpIhVxGT","executionInfo":{"status":"ok","timestamp":1654525449142,"user_tz":240,"elapsed":6512,"user":{"displayName":"Melanie Walsh","userId":"15679979807435926558"}},"colab":{"base_uri":"https://localhost:8080/","height":585},"outputId":"ecef69d8-0232-485f-db0d-5ae2734eb6f8"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["\n","
\n",""],"text/plain":["alt.Chart(...)"]},"metadata":{},"execution_count":47}]},{"cell_type":"markdown","source":["With your group, pick a word or words to examine."],"metadata":{"id":"AQDFJKZ84bbN"}},{"cell_type":"code","source":["# List of keywords that you want to compare\n","keywords =\n","\n","# How to color the points in the plot. The other option is \"period\" for time period\n","color_by = 'word'\n","\n","# Get all word positions\n","word_positions = get_word_positions(keywords)\n","\n","# Get all contexts around the words\n","keyword_contexts = []\n","keyword_contexts_tokens = []\n","words = []\n","\n","for position in word_positions:\n"," words.append(word_lookup[all_word_ids[position]])\n"," keyword_contexts.append(get_context_clean(position))\n"," keyword_contexts_tokens.append(get_context(position))\n","\n","# Reduce word vectors with PCA\n","pca = PCA(n_components=2)\n","pca.fit(all_word_vectors[word_positions,:].T)\n","\n","# Make a DataFrame with PCA results\n","df_to_plot = pd.DataFrame({\"x\": pca.components_[0,:],\n"," \"y\": pca.components_[1,:],\n"," \"context\": keyword_contexts,\n"," \"tokens\": keyword_contexts_tokens,\n"," \"word\": words})\n","# Match original text and metadata\n","df_to_plot[['stars', 'brand', 'name', 'type']] = df_to_plot.apply(find_original_review, axis='columns', result_type='expand')\n","\n","# Rename columns so that the context shows up as the \"title\" in the tooltip (bigger and bolded)\n","# df = df.rename(columns={'brand': 'brand', 'name': 'title'})\n","\n","# Make the plot\n","alt.Chart(df_to_plot, title=f\"Word Similarity: {', '.join(keywords).title()}\").mark_circle(size=200).encode(\n"," alt.X('x',\n"," scale=alt.Scale(zero=False)\n"," ), y=\"y\",\n"," color= color_by,\n"," # href=\"link\",\n"," tooltip=['word', 'context', 'type', 'brand', 'name', 'stars']\n"," ).interactive().properties(\n"," width=500,\n"," height=500\n",")"],"metadata":{"id":"t3MgT-o74afJ"},"execution_count":null,"outputs":[]}]}