{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "iGB4oSeeEYGE" }, "source": [ "# NLP for Data Science using Spacy & Co.\n", "\n", "Many language processing tasks are already covered in [Spacy 101 - Everything you need to know](https://spacy.io/usage/spacy-101).\n", "\n", "Spacy is an open source library made by the company [explosion.ai](https://explosion.ai/)." ] }, { "cell_type": "markdown", "metadata": { "id": "xlZptrO5lMRY" }, "source": [ "## Lemmatization, Part-of-speech tagging, and dependency parsing\n", "\n", "Many language analysis tasks involve a whole pipeline of models. Spacy models are trained to perform a combination of tasks, which is also a source of robustness of these models. When using these models, you can access multiple features produced by the model pipeline and use them for your own tasks." ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 394 }, "id": "zaRttKAZPAK0", "outputId": "9d595d86-0789-4662-e9b2-88156354bd72" }, "outputs": [ { "ename": "OSError", "evalue": "[E050] Can't find model 'en_core_web_sm'. It doesn't seem to be a Python package or a valid path to a data directory.", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mOSError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[1], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mspacy\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m nlp \u001b[38;5;241m=\u001b[39m \u001b[43mspacy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43men_core_web_sm\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 4\u001b[0m doc \u001b[38;5;241m=\u001b[39m nlp(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mApple is looking at buying U.K. startup for $1 billion\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 6\u001b[0m pd\u001b[38;5;241m.\u001b[39mDataFrame(((token\u001b[38;5;241m.\u001b[39mtext, token\u001b[38;5;241m.\u001b[39mlemma_, token\u001b[38;5;241m.\u001b[39mpos_, token\u001b[38;5;241m.\u001b[39mtag_, token\u001b[38;5;241m.\u001b[39mdep_,\n\u001b[1;32m 7\u001b[0m token\u001b[38;5;241m.\u001b[39mshape_, token\u001b[38;5;241m.\u001b[39mis_alpha, token\u001b[38;5;241m.\u001b[39mis_stop) \u001b[38;5;28;01mfor\u001b[39;00m token \u001b[38;5;129;01min\u001b[39;00m doc),\n\u001b[1;32m 8\u001b[0m columns\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mText\tLemma\tPOS\tTag\tDep\tShape\talpha\tstop\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39msplit())\n", "File \u001b[0;32m~/micromamba/lib/python3.11/site-packages/spacy/__init__.py:51\u001b[0m, in \u001b[0;36mload\u001b[0;34m(name, vocab, disable, enable, exclude, config)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload\u001b[39m(\n\u001b[1;32m 28\u001b[0m name: Union[\u001b[38;5;28mstr\u001b[39m, Path],\n\u001b[1;32m 29\u001b[0m \u001b[38;5;241m*\u001b[39m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 34\u001b[0m config: Union[Dict[\u001b[38;5;28mstr\u001b[39m, Any], Config] \u001b[38;5;241m=\u001b[39m util\u001b[38;5;241m.\u001b[39mSimpleFrozenDict(),\n\u001b[1;32m 35\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Language:\n\u001b[1;32m 36\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Load a spaCy model from an installed package or a local path.\u001b[39;00m\n\u001b[1;32m 37\u001b[0m \n\u001b[1;32m 38\u001b[0m \u001b[38;5;124;03m name (str): Package name or model path.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;124;03m RETURNS (Language): The loaded nlp object.\u001b[39;00m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 51\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mutil\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 52\u001b[0m \u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 53\u001b[0m \u001b[43m \u001b[49m\u001b[43mvocab\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvocab\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 54\u001b[0m \u001b[43m \u001b[49m\u001b[43mdisable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdisable\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 55\u001b[0m \u001b[43m \u001b[49m\u001b[43menable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menable\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 56\u001b[0m \u001b[43m \u001b[49m\u001b[43mexclude\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mexclude\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 57\u001b[0m \u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 58\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/micromamba/lib/python3.11/site-packages/spacy/util.py:472\u001b[0m, in \u001b[0;36mload_model\u001b[0;34m(name, vocab, disable, enable, exclude, config)\u001b[0m\n\u001b[1;32m 470\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m OLD_MODEL_SHORTCUTS:\n\u001b[1;32m 471\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mIOError\u001b[39;00m(Errors\u001b[38;5;241m.\u001b[39mE941\u001b[38;5;241m.\u001b[39mformat(name\u001b[38;5;241m=\u001b[39mname, full\u001b[38;5;241m=\u001b[39mOLD_MODEL_SHORTCUTS[name])) \u001b[38;5;66;03m# type: ignore[index]\u001b[39;00m\n\u001b[0;32m--> 472\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mIOError\u001b[39;00m(Errors\u001b[38;5;241m.\u001b[39mE050\u001b[38;5;241m.\u001b[39mformat(name\u001b[38;5;241m=\u001b[39mname))\n", "\u001b[0;31mOSError\u001b[0m: [E050] Can't find model 'en_core_web_sm'. It doesn't seem to be a Python package or a valid path to a data directory." ] } ], "source": [ "import spacy\n", "\n", "# prepare this model download via: python -m spacy download en_core_web_sm\n", "nlp = spacy.load(\"en_core_web_sm\")\n", "doc = nlp(\"Apple is looking at buying U.K. startup for $1 billion\")\n", "\n", "pd.DataFrame(((token.text, token.lemma_, token.pos_, token.tag_, token.dep_,\n", " token.shape_, token.is_alpha, token.is_stop) for token in doc),\n", " columns=\"Text\tLemma\tPOS\tTag\tDep\tShape\talpha\tstop\".split())\n" ] }, { "cell_type": "markdown", "metadata": { "id": "Q6vCayK4PEKs" }, "source": [ "## Named entity resolution (NER)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 219 }, "id": "ezcO4rx6O_ZO", "outputId": "cda6fb7e-6cd4-4355-ed99-d95861cceb9c" }, "outputs": [ { "data": { "text/html": [ "\n", "
\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
entitystartendtype
0Apple05ORG
1U.K.2731GPE
2$1 billion4454MONEY
\n", "
\n", " \n", " \n", " \n", "\n", " \n", "
\n", "
\n", " " ], "text/plain": [ " entity start end type\n", "0 Apple 0 5 ORG\n", "1 U.K. 27 31 GPE\n", "2 $1 billion 44 54 MONEY" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "### Entity highlighting using displacy" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", "\n", " Apple\n", " ORG\n", "\n", " is looking at buying \n", "\n", " U.K.\n", " GPE\n", "\n", " startup for \n", "\n", " $1 billion\n", " MONEY\n", "\n", "
" ], "text/plain": [ "" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import spacy\n", "from spacy import displacy\n", "from IPython.display import HTML, Markdown\n", "import pandas as pd\n", "\n", "nlp = spacy.load(\"en_core_web_sm\")\n", "doc = nlp(\"Apple is looking at buying U.K. startup for $1 billion\")\n", "\n", "def make_ents():\n", " for ent in doc.ents:\n", " yield (ent.text, ent.start_char, ent.end_char, ent.label_)\n", "\n", "display(pd.DataFrame(make_ents(), columns=\"entity start end type\".split()))\n", "\n", "# aside: some notebook TOC generators (e.g. Jupyter, but not Colab) pick up generated headings\n", "display(Markdown(\"### Entity highlighting using displacy\"))\n", "\n", "HTML(displacy.render(doc,\n", " #style='dep',\n", " style='ent',\n", " ))" ] }, { "cell_type": "markdown", "metadata": { "id": "ACeYqU2-DNBJ" }, "source": [ "## Constituency Parsing\n", "\n", "To generate the constituency tree, a more compehensive library is needed, constituent_treelib by Oren Halvani), that uses spacy as a feature extractor." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "hjKQuY_LCkkv" }, "outputs": [], "source": [ "#!pip install constituent-treelib" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "YNAuNNRTCo_O", "outputId": "eb366071-6106-43c4-9261-2905fdc49b21" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[nltk_data] Downloading package benepar_en3 to /usr/share/nltk_data...\n", "[nltk_data] Package benepar_en3 is already up-to-date!\n" ] } ], "source": [ "from constituent_treelib import ConstituentTree, BracketedTree, Language, Structure\n", "\n", "# Define the language for the sentence as well as for the spaCy and benepar models\n", "language = Language.English\n", "\n", "# Define which specific SpaCy model should be used (default is Medium)\n", "spacy_model_size = ConstituentTree.SpacyModelSize.Medium\n", "\n", "# Create the pipeline (note, the required models will be downloaded and installed automatically)\n", "nlp = ConstituentTree.create_pipeline(language, spacy_model_size)\n", "\n", "without_token_leaves = ConstituentTree(sentence, nlp, Structure.WithoutTokenLeaves)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3RwOX69WDRwX" }, "outputs": [], "source": [ "# Raw sentence\n", "sentence = 'You must construct additional pylons!'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "zlJhyEwatqHA" }, "outputs": [], "source": [ "\n", "# Parsed sentence wrapped as a BracketedTree object\n", "bracketed_tree_string = '(S (NP (PRP You)) (VP (MD must) (VP (VB construct) (NP (JJ additional) (NNS pylons)))) (. !))'\n", "bracketed_sentence = BracketedTree(bracketed_tree_string)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nSL1UlMQEA13" }, "outputs": [], "source": [ "without_token_leaves = ConstituentTree(sentence, nlp, Structure.WithoutTokenLeaves)\n", "\n", "#without_inner_postag_nodes = ConstituentTree(sentence, nlp, Structure.WithoutPostagNodes)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 238 }, "id": "E1ZB-a1CEIL3", "outputId": "3ea37ddd-e79b-48be-8f96-a7d00080c88b" }, "outputs": [ { "data": { "image/svg+xml": [ "SNPPRPVPMDVPVBNPJJNNS." ], "text/plain": [ "" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "without_token_leaves" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "OR6YhQd4EQ_J", "outputId": "401971ec-055f-4626-86e7-f711733afcda" }, "outputs": [ { "data": { "text/plain": [ "{'VP': ['must construct additional pylons', 'construct additional pylons'],\n", " 'S': ['You must construct additional pylons !'],\n", " 'NP': ['additional pylons']}" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pos_phrases = without_token_leaves.extract_all_phrases()\n", "pos_phrases" ] } ], "metadata": { "colab": { "provenance": [], "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.4" } }, "nbformat": 4, "nbformat_minor": 0 }