example2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPDLmJYdu7N3SB0Kh/6Ju79",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/david4096/huggingface-rdf/blob/main/example2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Use Huggingface Croissant ML via SPARQL\n",
        "\n",
        "Downloads a turtle file generated by [huggingface-rdf](https://github.com/david4096/huggingface-rdf) and sets up the environment so you can start write queries against it, or add other graphs :)"
      ],
      "metadata": {
        "id": "h0EoqDU1687P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install rdflib"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-HjNsKmS7Ick",
        "outputId": "e9973ce4-390b-4011-86d4-405ec7f6d776"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting rdflib\n",
            "  Downloading rdflib-7.1.1-py3-none-any.whl.metadata (11 kB)\n",
            "Collecting isodate<1.0.0,>=0.7.2 (from rdflib)\n",
            "  Downloading isodate-0.7.2-py3-none-any.whl.metadata (11 kB)\n",
            "Requirement already satisfied: pyparsing<4,>=2.1.0 in /usr/local/lib/python3.10/dist-packages (from rdflib) (3.2.0)\n",
            "Downloading rdflib-7.1.1-py3-none-any.whl (562 kB)\n",
            "\u001b[?25l   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/562.4 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K   \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━\u001b[0m \u001b[32m450.6/562.4 kB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m562.4/562.4 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading isodate-0.7.2-py3-none-any.whl (22 kB)\n",
            "Installing collected packages: isodate, rdflib\n",
            "Successfully installed isodate-0.7.2 rdflib-7.1.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Download a premade turtle file\n",
        "!wget https://github.com/david4096/huggingface-rdf/releases/download/data-release/huggingface.ttl"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LedMKJVo7kOz",
        "outputId": "c1a6cae2-c4e8-438b-8c90-005b08990609"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-11-04 16:07:38--  https://github.com/david4096/huggingface-rdf/releases/download/data-release/huggingface.ttl\n",
            "Resolving github.com (github.com)... 140.82.113.4\n",
            "Connecting to github.com (github.com)|140.82.113.4|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://objects.githubusercontent.com/github-production-release-asset-2e65be/848608772/85aedd13-8b02-4b23-865a-7472a7698ca9?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=releaseassetproduction%2F20241104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20241104T160738Z&X-Amz-Expires=300&X-Amz-Signature=d0bfa01d922e9a85948f9c2000a837f55a872e9c9638e805f28fbe2bb5c8a014&X-Amz-SignedHeaders=host&response-content-disposition=attachment%3B%20filename%3Dhuggingface.ttl&response-content-type=application%2Foctet-stream [following]\n",
            "--2024-11-04 16:07:38--  https://objects.githubusercontent.com/github-production-release-asset-2e65be/848608772/85aedd13-8b02-4b23-865a-7472a7698ca9?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=releaseassetproduction%2F20241104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20241104T160738Z&X-Amz-Expires=300&X-Amz-Signature=d0bfa01d922e9a85948f9c2000a837f55a872e9c9638e805f28fbe2bb5c8a014&X-Amz-SignedHeaders=host&response-content-disposition=attachment%3B%20filename%3Dhuggingface.ttl&response-content-type=application%2Foctet-stream\n",
            "Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.110.133, 185.199.111.133, 185.199.109.133, ...\n",
            "Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.110.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 22698390 (22M) [application/octet-stream]\n",
            "Saving to: ‘huggingface.ttl’\n",
            "\n",
            "huggingface.ttl     100%[===================>]  21.65M  81.8MB/s    in 0.3s    \n",
            "\n",
            "2024-11-04 16:07:39 (81.8 MB/s) - ‘huggingface.ttl’ saved [22698390/22698390]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import rdflib\n",
        "g = rdflib.Graph()\n",
        "g.parse('huggingface.ttl')\n",
        "test_query = \"\"\"\n",
        "SELECT DISTINCT ?b WHERE {?a ?b ?c}\"\"\"\n",
        "\n",
        "qres = g.query(test_query)\n",
        "for row in qres:\n",
        "    print(f\"{row.b}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k5TZfoev7Sel",
        "outputId": "6aed00f8-5d4b-4c2c-a271-7d72ae177c51"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "http://mlcommons.org/croissant/source\n",
            "https://schema.org/keywords\n",
            "http://www.w3.org/1999/02/22-rdf-syntax-ns#type\n",
            "http://mlcommons.org/croissant/column\n",
            "http://mlcommons.org/croissant/dataType\n",
            "http://mlcommons.org/croissant/fileSet\n",
            "https://schema.org/name\n",
            "http://mlcommons.org/croissant/extract\n",
            "https://schema.org/description\n",
            "https://schema.org/containedIn\n",
            "https://schema.org/encodingFormat\n",
            "https://schema.org/contentUrl\n",
            "https://schema.org/creator\n",
            "http://mlcommons.org/croissant/field\n",
            "https://schema.org/distribution\n",
            "https://schema.org/url\n",
            "http://mlcommons.org/croissant/includes\n",
            "https://schema.org/alternateName\n",
            "https://schema.org/sameAs\n",
            "http://mlcommons.org/croissant/recordSet\n",
            "http://mlcommons.org/croissant/repeated\n",
            "https://schema.org/license\n",
            "http://purl.org/dc/terms/conformsTo\n",
            "http://mlcommons.org/croissant/jsonPath\n",
            "http://mlcommons.org/croissant/transform\n",
            "https://schema.org/identifier\n",
            "https://schema.org/sha256\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## What next?\n",
        "\n",
        "* Demonstrate valuable queries\n",
        "* Identifying public datasets that may be interesting to join on (e.g. wikidata for institutions to join on enabling geospatial type queries)\n",
        "* Enrich the metadata by adding triplets that convert freetext elements into controlled categories (EDAM via LLM, another project here is working on something similar maybe chance to collaborate)\n",
        "* Do similar for Kaggle, which also does Croissant and compare quality\n",
        "* Connect to citation graph via author by linking ORCIDs\n",
        "* Identify biology related datasets in huggingface and demonstrating value of Croissant for those cases"
      ],
      "metadata": {
        "id": "kRpyjnt48Epo"
      }
    }
  ]
}