classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "717c71de",
   "metadata": {},
   "outputs": [],
   "source": [
    "# custom script for deleting files from previous run \n",
    "%run /Users/Cadence/Desktop/pdf_classifier/scripts/cleanup.py\n",
    "import os, glob, re, io, random, gensim, smart_open, logging, collections, sys\n",
    "#glob for 讀取檔案路徑\n",
    "\n",
    "from PyPDF2 import PdfReader\n",
    "#PyPDF2--text extraction, pdfminer--error handling fallback\n",
    "\n",
    "from pdfminer.high_level import extract_text as pdfminer_extraction\n",
    "from pdfminer.high_level import extract_text_to_fp\n",
    "from pdfminer.layout import LAParams\n",
    "from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n",
    "from gensim.test.utils import get_tmpfile\n",
    "\n",
    "pdfReaders = []\n",
    "pdfFiles = []\n",
    "docLabels = []\n",
    "\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "rootDir = \"/Users/Cadence/Desktop/pdf_classifier\"\n",
    "pdfReadRootDir =  rootDir + '/pdf_test_coffee'\n",
    "txtExtractDir = rootDir + '/txt-extractions'\n",
    "modelDataDir = rootDir + '/model_data'\n",
    "testsDir = rootDir + '/tests'\n",
    "\n",
    "\n",
    "class bcolors:\n",
    "    HEADER = '\\033[95m'\n",
    "    OKBLUE = '\\033[94m'\n",
    "    OKCYAN = '\\033[96m'\n",
    "    OKGREEN = '\\033[92m'\n",
    "    WARNING = '\\033[93m'\n",
    "    FAIL = '\\033[91m'\n",
    "    ENDC = '\\033[0m'\n",
    "    BOLD = '\\033[1m'\n",
    "    UNDERLINE = '\\033[4m'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "980957ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "###### PART 1 - READ, EXTRACT, TRAIN AND ASSESS ######"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b6cacd12",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading pdfs in/Users/Cadence/Desktop/pdf_classifier/pdf_test_coffee (including subdirectories)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-26 15:00:10,326 : WARNING : Previous trailer can not be read (\"invalid literal for int() with base 10: b'n'\",)\n",
      "2023-05-26 15:00:10,329 : WARNING : Previous trailer can not be read (\"invalid literal for int() with base 10: b'n'\",)\n",
      "2023-05-26 15:00:13,951 : WARNING : incorrect startxref pointer(1)\n",
      "2023-05-26 15:00:13,985 : WARNING : incorrect startxref pointer(1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92mpdf files read: 496\u001b[0m\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"reading pdfs in\" + str(pdfReadRootDir) + \" (including subdirectories)\")\n",
    "def read_files():\n",
    "    os.chdir(pdfReadRootDir)\n",
    "    counter = 0  \n",
    "    for file in glob.glob(\"**/*.pdf\", recursive=True):\n",
    "        try:\n",
    "            counter += 1  \n",
    "            pdfFiles.append(file)\n",
    "            pdf_reader = PdfReader(file)\n",
    "            pdfReaders.append(PdfReader(file))\n",
    "        #有問題就輸出錯誤訊息，跳過檔案    \n",
    "        except Exception as e:\n",
    "            print(bcolors.FAIL + \"error: \" + file + \" is unreadable. Skipping file\" + bcolors.ENDC)\n",
    "            print(\"Exception:\", str(e))\n",
    "            continue\n",
    "    print(bcolors.OKGREEN + \"pdf files read: \" + str(counter) + bcolors.ENDC)  # 輸出讀取的檔案數量\n",
    "    print()\n",
    "\n",
    "read_files()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "edb41a39",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting pdfs to text files (duplicate pdfs are handled automagically)\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-26 15:01:50,555 : WARNING : Object 153 0 found\n",
      "2023-05-26 15:01:50,561 : WARNING : Object 151 0 found\n",
      "2023-05-26 15:01:50,565 : WARNING : Object 1 0 found\n",
      "2023-05-26 15:01:50,569 : WARNING : Object 15 0 found\n",
      "2023-05-26 15:01:50,573 : WARNING : Object 23 0 found\n",
      "2023-05-26 15:01:50,578 : WARNING : Object 29 0 found\n",
      "2023-05-26 15:01:50,583 : WARNING : Object 37 0 found\n",
      "2023-05-26 15:01:50,589 : WARNING : Object 51 0 found\n",
      "2023-05-26 15:01:50,630 : WARNING : Object 12 0 found\n",
      "2023-05-26 15:01:50,636 : WARNING : Object 92 0 found\n",
      "2023-05-26 15:01:50,641 : WARNING : Object 81 0 found\n",
      "2023-05-26 15:01:50,645 : WARNING : Object 87 0 found\n",
      "2023-05-26 15:01:50,651 : WARNING : Object 82 0 found\n",
      "2023-05-26 15:01:50,657 : WARNING : Object 86 0 found\n",
      "2023-05-26 15:01:50,662 : WARNING : Object 89 0 found\n",
      "2023-05-26 15:01:50,667 : WARNING : Object 90 0 found\n",
      "2023-05-26 15:01:50,672 : WARNING : Object 83 0 found\n",
      "2023-05-26 15:01:50,675 : WARNING : Object 13 0 found\n",
      "2023-05-26 15:01:50,692 : WARNING : Object 14 0 found\n",
      "2023-05-26 15:01:50,697 : WARNING : Object 18 0 found\n",
      "2023-05-26 15:01:50,702 : WARNING : Object 93 0 found\n",
      "2023-05-26 15:01:50,705 : WARNING : Object 20 0 found\n",
      "2023-05-26 15:01:50,710 : WARNING : Object 22 0 found\n",
      "2023-05-26 15:01:50,714 : WARNING : Object 19 0 found\n",
      "2023-05-26 15:01:50,747 : WARNING : Object 26 0 found\n",
      "2023-05-26 15:01:50,750 : WARNING : Object 27 0 found\n",
      "2023-05-26 15:01:50,767 : WARNING : Object 28 0 found\n",
      "2023-05-26 15:01:50,773 : WARNING : Object 34 0 found\n",
      "2023-05-26 15:01:50,779 : WARNING : Object 35 0 found\n",
      "2023-05-26 15:01:50,801 : WARNING : Object 36 0 found\n",
      "2023-05-26 15:01:50,807 : WARNING : Object 45 0 found\n",
      "2023-05-26 15:01:50,814 : WARNING : Object 94 0 found\n",
      "2023-05-26 15:01:50,818 : WARNING : Object 47 0 found\n",
      "2023-05-26 15:01:50,822 : WARNING : Object 50 0 found\n",
      "2023-05-26 15:01:50,828 : WARNING : Object 49 0 found\n",
      "2023-05-26 15:01:50,833 : WARNING : Object 46 0 found\n",
      "2023-05-26 15:01:50,862 : WARNING : Object 73 0 found\n",
      "2023-05-26 15:01:50,867 : WARNING : Object 96 0 found\n",
      "2023-05-26 15:01:50,871 : WARNING : Object 75 0 found\n",
      "2023-05-26 15:01:50,876 : WARNING : Object 78 0 found\n",
      "2023-05-26 15:01:50,882 : WARNING : Object 77 0 found\n",
      "2023-05-26 15:01:50,887 : WARNING : Object 95 0 found\n",
      "2023-05-26 15:01:50,893 : WARNING : Object 80 0 found\n",
      "2023-05-26 15:01:50,898 : WARNING : Object 74 0 found\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-26 15:03:33,737 : WARNING : Multiple definitions in dictionary at byte 0x2f692 for key /MediaBox\n",
      "2023-05-26 15:03:33,739 : WARNING : Multiple definitions in dictionary at byte 0x2f99c for key /MediaBox\n",
      "2023-05-26 15:03:33,742 : WARNING : Multiple definitions in dictionary at byte 0x2fbc6 for key /MediaBox\n",
      "2023-05-26 15:03:33,744 : WARNING : Multiple definitions in dictionary at byte 0x2fedf for key /MediaBox\n",
      "2023-05-26 15:03:33,746 : WARNING : Multiple definitions in dictionary at byte 0x30208 for key /MediaBox\n",
      "2023-05-26 15:03:33,749 : WARNING : Multiple definitions in dictionary at byte 0x304b9 for key /MediaBox\n",
      "2023-05-26 15:03:33,751 : WARNING : Multiple definitions in dictionary at byte 0x30772 for key /MediaBox\n",
      "2023-05-26 15:03:33,754 : WARNING : Multiple definitions in dictionary at byte 0x30a4b for key /MediaBox\n",
      "2023-05-26 15:03:33,757 : WARNING : Multiple definitions in dictionary at byte 0x30f25 for key /MediaBox\n",
      "2023-05-26 15:03:33,759 : WARNING : Multiple definitions in dictionary at byte 0x3119e for key /MediaBox\n",
      "2023-05-26 15:03:33,761 : WARNING : Multiple definitions in dictionary at byte 0x31427 for key /MediaBox\n",
      "2023-05-26 15:03:33,762 : WARNING : Multiple definitions in dictionary at byte 0x315c0 for key /MediaBox\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[91merror: Metadata or title is missing for the PDF document. Skipping file\u001b[0m\n",
      "\u001b[92mpdf extraction complete\u001b[0m\n",
      "Successfully extracted text from 424 documents\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "# 定義無效字符\n",
    "invalid_chars = r'[<>:\"/\\\\|?*]'\n",
    "\n",
    "def sanitize_filename(filename):\n",
    "    # 替換無效字符\n",
    "    return re.sub(invalid_chars, '_', filename)\n",
    "\n",
    "# extract text from pdfs to designated directory and save as txt files.\n",
    "def extract_to_txt():\n",
    "    print(\"Extracting pdfs to text files (duplicate pdfs are handled automagically)\")\n",
    "    os.chdir(txtExtractDir)\n",
    "    counter = 0\n",
    "    successful_count = 0  \n",
    "    for i in pdfReaders:\n",
    "        counter += 1\n",
    "        if i.metadata is None or i.metadata.title is None:\n",
    "            print(bcolors.FAIL + \"error: \" + \"Metadata or title is missing for the PDF document. Skipping file\" + bcolors.ENDC)\n",
    "            continue\n",
    "            \n",
    "        \n",
    "        title = str(i.metadata.title) if i.metadata.title is not None else \"\"\n",
    "        filename = sanitize_filename(title)\n",
    "        text = \"\"\n",
    "        with open(filename + \".txt\", 'w', encoding=\"utf-8\") as file:\n",
    "            docLabels.append(i.metadata.title)\n",
    "            try:\n",
    "                for j in range(len(i.pages)):\n",
    "                    text += i.pages[j].extract_text()\n",
    "                successful_count += 1  \n",
    "            except Exception as exc:\n",
    "                print(bcolors.FAIL + \"error: \" + \"\\\"\" + str(i.metadata.title()) + \"\\\"\" + \" not extractable with PyPDF2, trying with pdfminer\" + bcolors.ENDC)\n",
    "                print()\n",
    "                # format txt file so that each document is one one line (doc2vec requirement)\n",
    "                text += pdfminer_extraction(pdfReadRootDir + \"/\" + pdfFiles[counter])\n",
    "                \n",
    "            # 一次性的文本連接與處裡\n",
    "            text = \"\".join(line.strip(\"\\n\") for line in text)\n",
    "            file.write(text)\n",
    "    \n",
    "    print(bcolors.OKGREEN + \"pdf extraction complete\" + bcolors.ENDC)\n",
    "    print(\"Successfully extracted text from\", successful_count, \"documents\")\n",
    "    \n",
    "extract_to_txt()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1264dcf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate a training corpus from all txt files found in designated directory\n",
    "class CorpusGen(object):\n",
    "    def __init__(self, dirname):\n",
    "        self.dirname = dirname\n",
    "\n",
    "    def __iter__(self, tokens_only=False):\n",
    "        counter = 0\n",
    "        for fname in os.listdir(self.dirname):\n",
    "            \n",
    "            with smart_open.open(fname, encoding=\"iso-8859-1\") as f:\n",
    "                for i, line in enumerate(f):\n",
    "                    tokens = gensim.utils.simple_preprocess(line, min_len=3, max_len=20, deacc=True)\n",
    "                    if tokens_only:\n",
    "                        yield tokens\n",
    "                    else:\n",
    "                        yield gensim.models.doc2vec.TaggedDocument(tokens, [counter])\n",
    "            counter += 1\n",
    "        \n",
    "trainCorpus = list(CorpusGen(txtExtractDir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9b4494fc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# save the entire corpus to a txt file\n",
    "# open('text.txt', 'w', encoding='UTF-8')\n",
    "\n",
    "with open(modelDataDir + \"/train-corpus.txt\", 'w', encoding='UTF-8') as file:\n",
    "    file.write(str(trainCorpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d81291b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-26 15:23:47,147 : INFO : Doc2Vec lifecycle event {'params': 'Doc2Vec<dm/m,d50,n5,w5,mc2,s0.001,t3>', 'datetime': '2023-05-26T15:23:47.146660', 'gensim': '4.3.0', 'python': '3.10.9 | packaged by Anaconda, Inc. | (main, Mar  1 2023, 18:18:15) [MSC v.1916 64 bit (AMD64)]', 'platform': 'Windows-10-10.0.22621-SP0', 'event': 'created'}\n",
      "2023-05-26 15:23:47,148 : INFO : collecting all words and their counts\n",
      "2023-05-26 15:23:47,149 : INFO : PROGRESS: at example #0, processed 0 words (0 words/s), 0 word types, 0 tags\n",
      "2023-05-26 15:23:47,671 : INFO : collected 168475 word types and 317 unique tags from a corpus of 385 examples and 2485925 words\n",
      "2023-05-26 15:23:47,672 : INFO : Creating a fresh vocabulary\n",
      "2023-05-26 15:23:48,003 : INFO : Doc2Vec lifecycle event {'msg': 'effective_min_count=2 retains 65913 unique words (39.12% of original 168475, drops 102562)', 'datetime': '2023-05-26T15:23:48.003690', 'gensim': '4.3.0', 'python': '3.10.9 | packaged by Anaconda, Inc. | (main, Mar  1 2023, 18:18:15) [MSC v.1916 64 bit (AMD64)]', 'platform': 'Windows-10-10.0.22621-SP0', 'event': 'prepare_vocab'}\n",
      "2023-05-26 15:23:48,004 : INFO : Doc2Vec lifecycle event {'msg': 'effective_min_count=2 leaves 2383363 word corpus (95.87% of original 2485925, drops 102562)', 'datetime': '2023-05-26T15:23:48.004909', 'gensim': '4.3.0', 'python': '3.10.9 | packaged by Anaconda, Inc. | (main, Mar  1 2023, 18:18:15) [MSC v.1916 64 bit (AMD64)]', 'platform': 'Windows-10-10.0.22621-SP0', 'event': 'prepare_vocab'}\n",
      "2023-05-26 15:23:48,422 : INFO : deleting the raw counts dictionary of 168475 items\n",
      "2023-05-26 15:23:48,427 : INFO : sample=0.001 downsamples 20 most-common words\n",
      "2023-05-26 15:23:48,429 : INFO : Doc2Vec lifecycle event {'msg': 'downsampling leaves estimated 2096423.7124415063 word corpus (88.0%% of prior 2383363)', 'datetime': '2023-05-26T15:23:48.429029', 'gensim': '4.3.0', 'python': '3.10.9 | packaged by Anaconda, Inc. | (main, Mar  1 2023, 18:18:15) [MSC v.1916 64 bit (AMD64)]', 'platform': 'Windows-10-10.0.22621-SP0', 'event': 'prepare_vocab'}\n",
      "2023-05-26 15:23:49,044 : INFO : estimated required memory for 65913 words and 50 dimensions: 59448500 bytes\n",
      "2023-05-26 15:23:49,045 : INFO : resetting layer weights\n",
      "2023-05-26 15:23:49,063 : INFO : Doc2Vec lifecycle event {'msg': 'training model with 3 workers on 65913 vocabulary and 50 features, using sg=0 hs=0 sample=0.001 negative=5 window=5 shrink_windows=True', 'datetime': '2023-05-26T15:23:49.063972', 'gensim': '4.3.0', 'python': '3.10.9 | packaged by Anaconda, Inc. | (main, Mar  1 2023, 18:18:15) [MSC v.1916 64 bit (AMD64)]', 'platform': 'Windows-10-10.0.22621-SP0', 'event': 'train'}\n",
      "2023-05-26 15:23:50,076 : INFO : EPOCH 0 - PROGRESS: at 73.51% examples, 1300628 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:23:50,525 : INFO : EPOCH 0: training on 2485925 raw words (1904784 effective words) took 1.5s, 1308674 effective words/s\n",
      "2023-05-26 15:23:51,534 : INFO : EPOCH 1 - PROGRESS: at 73.51% examples, 1303751 words/s, in_qsize 6, out_qsize 0\n",
      "2023-05-26 15:23:51,967 : INFO : EPOCH 1: training on 2485925 raw words (1904835 effective words) took 1.4s, 1325746 effective words/s\n",
      "2023-05-26 15:23:52,980 : INFO : EPOCH 2 - PROGRESS: at 71.95% examples, 1261053 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:23:53,468 : INFO : EPOCH 2: training on 2485925 raw words (1905007 effective words) took 1.5s, 1271779 effective words/s\n",
      "2023-05-26 15:23:54,477 : INFO : EPOCH 3 - PROGRESS: at 71.95% examples, 1266279 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:23:54,975 : INFO : EPOCH 3: training on 2485925 raw words (1904987 effective words) took 1.5s, 1268044 effective words/s\n",
      "2023-05-26 15:23:55,991 : INFO : EPOCH 4 - PROGRESS: at 71.95% examples, 1256968 words/s, in_qsize 5, out_qsize 1\n",
      "2023-05-26 15:23:56,487 : INFO : EPOCH 4: training on 2485925 raw words (1904845 effective words) took 1.5s, 1263959 effective words/s\n",
      "2023-05-26 15:23:57,500 : INFO : EPOCH 5 - PROGRESS: at 75.32% examples, 1331608 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:23:57,900 : INFO : EPOCH 5: training on 2485925 raw words (1904992 effective words) took 1.4s, 1352401 effective words/s\n",
      "2023-05-26 15:23:58,906 : INFO : EPOCH 6 - PROGRESS: at 74.81% examples, 1331407 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:23:59,324 : INFO : EPOCH 6: training on 2485925 raw words (1904541 effective words) took 1.4s, 1341127 effective words/s\n",
      "2023-05-26 15:24:00,337 : INFO : EPOCH 7 - PROGRESS: at 75.32% examples, 1333073 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:00,749 : INFO : EPOCH 7: training on 2485925 raw words (1904685 effective words) took 1.4s, 1340143 effective words/s\n",
      "2023-05-26 15:24:01,757 : INFO : EPOCH 8 - PROGRESS: at 71.95% examples, 1268484 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:02,238 : INFO : EPOCH 8: training on 2485925 raw words (1904671 effective words) took 1.5s, 1282631 effective words/s\n",
      "2023-05-26 15:24:03,248 : INFO : EPOCH 9 - PROGRESS: at 76.10% examples, 1359765 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:03,632 : INFO : EPOCH 9: training on 2485925 raw words (1904469 effective words) took 1.4s, 1370869 effective words/s\n",
      "2023-05-26 15:24:04,646 : INFO : EPOCH 10 - PROGRESS: at 75.32% examples, 1333122 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:05,067 : INFO : EPOCH 10: training on 2485925 raw words (1904373 effective words) took 1.4s, 1330390 effective words/s\n",
      "2023-05-26 15:24:06,075 : INFO : EPOCH 11 - PROGRESS: at 74.29% examples, 1321202 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:06,508 : INFO : EPOCH 11: training on 2485925 raw words (1904746 effective words) took 1.4s, 1327650 effective words/s\n",
      "2023-05-26 15:24:07,518 : INFO : EPOCH 12 - PROGRESS: at 69.87% examples, 1220337 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:08,052 : INFO : EPOCH 12: training on 2485925 raw words (1905232 effective words) took 1.5s, 1237974 effective words/s\n",
      "2023-05-26 15:24:09,069 : INFO : EPOCH 13 - PROGRESS: at 75.32% examples, 1332099 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:09,656 : INFO : EPOCH 13: training on 2485925 raw words (1904610 effective words) took 1.6s, 1193225 effective words/s\n",
      "2023-05-26 15:24:10,662 : INFO : EPOCH 14 - PROGRESS: at 65.71% examples, 1136165 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:11,335 : INFO : EPOCH 14: training on 2485925 raw words (1904562 effective words) took 1.7s, 1136893 effective words/s\n",
      "2023-05-26 15:24:12,342 : INFO : EPOCH 15 - PROGRESS: at 64.68% examples, 1117136 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:13,065 : INFO : EPOCH 15: training on 2485925 raw words (1904787 effective words) took 1.7s, 1102845 effective words/s\n",
      "2023-05-26 15:24:14,075 : INFO : EPOCH 16 - PROGRESS: at 68.57% examples, 1197267 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:14,627 : INFO : EPOCH 16: training on 2485925 raw words (1904940 effective words) took 1.6s, 1224377 effective words/s\n",
      "2023-05-26 15:24:15,634 : INFO : EPOCH 17 - PROGRESS: at 73.25% examples, 1295517 words/s, in_qsize 6, out_qsize 0\n",
      "2023-05-26 15:24:16,102 : INFO : EPOCH 17: training on 2485925 raw words (1905254 effective words) took 1.5s, 1295761 effective words/s\n",
      "2023-05-26 15:24:17,110 : INFO : EPOCH 18 - PROGRESS: at 72.99% examples, 1285189 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:17,566 : INFO : EPOCH 18: training on 2485925 raw words (1904508 effective words) took 1.5s, 1304812 effective words/s\n",
      "2023-05-26 15:24:18,572 : INFO : EPOCH 19 - PROGRESS: at 73.77% examples, 1312062 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:19,042 : INFO : EPOCH 19: training on 2485925 raw words (1904537 effective words) took 1.5s, 1293316 effective words/s\n",
      "2023-05-26 15:24:20,052 : INFO : EPOCH 20 - PROGRESS: at 71.95% examples, 1264156 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:20,518 : INFO : EPOCH 20: training on 2485925 raw words (1904562 effective words) took 1.5s, 1293655 effective words/s\n",
      "2023-05-26 15:24:21,525 : INFO : EPOCH 21 - PROGRESS: at 75.32% examples, 1344237 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:21,936 : INFO : EPOCH 21: training on 2485925 raw words (1904336 effective words) took 1.4s, 1349312 effective words/s\n",
      "2023-05-26 15:24:22,947 : INFO : EPOCH 22 - PROGRESS: at 71.95% examples, 1263489 words/s, in_qsize 6, out_qsize 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-26 15:24:23,400 : INFO : EPOCH 22: training on 2485925 raw words (1904986 effective words) took 1.5s, 1304427 effective words/s\n",
      "2023-05-26 15:24:24,413 : INFO : EPOCH 23 - PROGRESS: at 75.32% examples, 1333314 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:24,833 : INFO : EPOCH 23: training on 2485925 raw words (1904667 effective words) took 1.4s, 1333099 effective words/s\n",
      "2023-05-26 15:24:25,842 : INFO : EPOCH 24 - PROGRESS: at 70.13% examples, 1227841 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:26,368 : INFO : EPOCH 24: training on 2485925 raw words (1904806 effective words) took 1.5s, 1243600 effective words/s\n",
      "2023-05-26 15:24:27,381 : INFO : EPOCH 25 - PROGRESS: at 71.95% examples, 1261670 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:27,871 : INFO : EPOCH 25: training on 2485925 raw words (1904760 effective words) took 1.5s, 1270529 effective words/s\n",
      "2023-05-26 15:24:28,880 : INFO : EPOCH 26 - PROGRESS: at 72.99% examples, 1282475 words/s, in_qsize 6, out_qsize 0\n",
      "2023-05-26 15:24:29,345 : INFO : EPOCH 26: training on 2485925 raw words (1904704 effective words) took 1.5s, 1295432 effective words/s\n",
      "2023-05-26 15:24:30,353 : INFO : EPOCH 27 - PROGRESS: at 75.32% examples, 1340859 words/s, in_qsize 6, out_qsize 0\n",
      "2023-05-26 15:24:30,758 : INFO : EPOCH 27: training on 2485925 raw words (1905167 effective words) took 1.4s, 1351771 effective words/s\n",
      "2023-05-26 15:24:31,764 : INFO : EPOCH 28 - PROGRESS: at 74.55% examples, 1326518 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:32,192 : INFO : EPOCH 28: training on 2485925 raw words (1904793 effective words) took 1.4s, 1331949 effective words/s\n",
      "2023-05-26 15:24:33,201 : INFO : EPOCH 29 - PROGRESS: at 76.62% examples, 1370996 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:33,574 : INFO : EPOCH 29: training on 2485925 raw words (1905004 effective words) took 1.4s, 1382294 effective words/s\n",
      "2023-05-26 15:24:34,583 : INFO : EPOCH 30 - PROGRESS: at 73.25% examples, 1296793 words/s, in_qsize 6, out_qsize 0\n",
      "2023-05-26 15:24:35,035 : INFO : EPOCH 30: training on 2485925 raw words (1904608 effective words) took 1.5s, 1310304 effective words/s\n",
      "2023-05-26 15:24:36,044 : INFO : EPOCH 31 - PROGRESS: at 74.29% examples, 1322762 words/s, in_qsize 6, out_qsize 0\n",
      "2023-05-26 15:24:36,494 : INFO : EPOCH 31: training on 2485925 raw words (1905141 effective words) took 1.5s, 1313858 effective words/s\n",
      "2023-05-26 15:24:37,498 : INFO : EPOCH 32 - PROGRESS: at 68.83% examples, 1204070 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:38,050 : INFO : EPOCH 32: training on 2485925 raw words (1904596 effective words) took 1.6s, 1227379 effective words/s\n",
      "2023-05-26 15:24:39,062 : INFO : EPOCH 33 - PROGRESS: at 76.10% examples, 1355749 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:39,453 : INFO : EPOCH 33: training on 2485925 raw words (1904569 effective words) took 1.4s, 1361365 effective words/s\n",
      "2023-05-26 15:24:40,461 : INFO : EPOCH 34 - PROGRESS: at 74.81% examples, 1331000 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:40,895 : INFO : EPOCH 34: training on 2485925 raw words (1904885 effective words) took 1.4s, 1326146 effective words/s\n",
      "2023-05-26 15:24:41,904 : INFO : EPOCH 35 - PROGRESS: at 72.99% examples, 1284037 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:42,395 : INFO : EPOCH 35: training on 2485925 raw words (1905099 effective words) took 1.5s, 1274111 effective words/s\n",
      "2023-05-26 15:24:43,405 : INFO : EPOCH 36 - PROGRESS: at 65.71% examples, 1135170 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:44,139 : INFO : EPOCH 36: training on 2485925 raw words (1905281 effective words) took 1.7s, 1096759 effective words/s\n",
      "2023-05-26 15:24:45,147 : INFO : EPOCH 37 - PROGRESS: at 61.56% examples, 1046409 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:45,860 : INFO : EPOCH 37: training on 2485925 raw words (1904619 effective words) took 1.7s, 1109382 effective words/s\n",
      "2023-05-26 15:24:46,870 : INFO : EPOCH 38 - PROGRESS: at 68.83% examples, 1199785 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:47,449 : INFO : EPOCH 38: training on 2485925 raw words (1904882 effective words) took 1.6s, 1201508 effective words/s\n",
      "2023-05-26 15:24:48,458 : INFO : EPOCH 39 - PROGRESS: at 67.79% examples, 1180490 words/s, in_qsize 5, out_qsize 0\n",
      "2023-05-26 15:24:49,087 : INFO : EPOCH 39: training on 2485925 raw words (1904063 effective words) took 1.6s, 1166642 effective words/s\n",
      "2023-05-26 15:24:49,088 : INFO : Doc2Vec lifecycle event {'msg': 'training on 99437000 raw words (76190893 effective words) took 60.0s, 1269344 effective words/s', 'datetime': '2023-05-26T15:24:49.088714', 'gensim': '4.3.0', 'python': '3.10.9 | packaged by Anaconda, Inc. | (main, Mar  1 2023, 18:18:15) [MSC v.1916 64 bit (AMD64)]', 'platform': 'Windows-10-10.0.22621-SP0', 'event': 'train'}\n"
     ]
    }
   ],
   "source": [
    "# establish a model and build the vocab\n",
    "model = gensim.models.doc2vec.Doc2Vec(vector_size=50, min_count=2, epochs=40)\n",
    "model.build_vocab(trainCorpus)\n",
    "model.train(trainCorpus, total_examples=model.corpus_count, epochs=model.epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2de91152",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-26 15:26:28,003 : WARNING : attribute doc_count not present in KeyedVectors<vector_size=50, 317 keys>; will store in internal index_to_key order\n",
      "2023-05-26 15:26:28,004 : INFO : storing 317x50 projection weights into doc_tensor.w2v\n",
      "2023-05-26 15:26:28,040 : INFO : running ../scripts/word2vec2tensor.py -i doc_tensor.w2v -o pdf_plot\n",
      "2023-05-26 15:26:28,041 : INFO : loading projection weights from doc_tensor.w2v\n",
      "2023-05-26 15:26:28,067 : INFO : KeyedVectors lifecycle event {'msg': 'loaded (317, 50) matrix of type float32 from doc_tensor.w2v', 'binary': False, 'encoding': 'utf8', 'datetime': '2023-05-26T15:26:28.067087', 'gensim': '4.3.0', 'python': '3.10.9 | packaged by Anaconda, Inc. | (main, Mar  1 2023, 18:18:15) [MSC v.1916 64 bit (AMD64)]', 'platform': 'Windows-10-10.0.22621-SP0', 'event': 'load_word2vec_format'}\n",
      "2023-05-26 15:26:28,081 : INFO : 2D tensor file saved to pdf_plot_tensor.tsv\n",
      "2023-05-26 15:26:28,082 : INFO : Tensor metadata file saved to pdf_plot_metadata.tsv\n",
      "2023-05-26 15:26:28,082 : INFO : finished running word2vec2tensor.py\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "02Pawłowska123.txt\n",
      "06_Novak_RGleb_03_2020.indd.txt\n",
      "101年度專題討論1-318(付印).indd.txt\n",
      "42 (3) 2021.txt\n",
      "A Comparative Photographic Review on Higher Plants and Macro-Fungi_ A Soil Restoration for Sustainable Production of Food and Energy.txt\n",
      "A generalizable framework for spatially explicit exploration of soil organic carbon sequestration on global marginal land.txt\n",
      "A meta-analysis of the effect of organic and mineral fertilizers on soil microbial diversity.txt\n",
      "A meta‐analysis of mycorrhizal responses to nitrogen, phosphorus, and atmospheric CO2 in field studies.txt\n",
      "A review and meta-analysis of mitigation measures for nitrous oxide emissions from crop residues.txt\n",
      "A review of trends and drivers of greenhouse gas emissions by sector from 1990 to 2018.txt\n",
      "A Transferable Learning Classification Model and Carbon Sequestration Estimation of Crops in Farmland Ecosystem.txt\n",
      "A “footprint” of plant carbon fixation cycle functions during the development of a heterotrophic fungus.txt\n",
      "ADD_TX_1~ABS_ABSH_AT_ADD_TX_2~ABS_ABSH_AT.txt\n",
      "Agro-environmental characterization of biochar issued from crop wastes in the humid forest zone of Cameroon.txt\n",
      "Application of Porous Materials for CO2 Reutilization_ A Review.txt\n",
      "Arbuscular mycorrhizal fungi enhance soil carbon sequestration in the coalfields, northwest China.txt\n",
      "Arbuscular Mycorrhizal Fungi in Agriculture.txt\n",
      "Arbuscular Mycorrhizal Fungi Increase Organic Carbon Decomposition Under Elevated CO2.txt\n",
      "Article.txt\n",
      "Assessing the impact of within crop heterogeneity (‘patchiness’) in young Miscanthus × giganteus fields on economic feasibility and soil carbon sequestration.txt\n",
      "Assessment of the Carbon Budget of Local Governments in South Korea.txt\n",
      "Autotrophic carbon fixation in archaea.txt\n",
      "Balancing a budget or running a deficit_ The offset regime of carbon removal and solar geoengineering under a carbon budget.txt\n",
      "Belowground carbon trade among tall trees in a temperate forest.txt\n",
      "Biochar for agronomy, animal farming, anaerobic digestion, composting, water treatment, soil remediation, construction, energy storage, and carbon sequestration_ a review.txt\n",
      "Biochar in agriculture – A systematic review of 26 global meta‐analyses.txt\n",
      "Biochar in climate change mitigation.txt\n",
      "Biomass Production and Carbon Sequestration Potential of Different Agroforestry Systems in India_ A Critical Review.txt\n",
      "Biotechnological Advances to Improve Abiotic Stress Tolerance in Crops.txt\n",
      "Biotic Interactions in Soil are Underestimated Drivers of Microbial Carbon Use Efficiency.txt\n",
      "Bio‐organic soil amendment promotes the suppression of Ralstonia solanacearum by inducing changes in the functionality and composition of rhizosphere bacterial communities.txt\n",
      "borzecka.vp.txt\n",
      "Both catabolic and anabolic heterotrophic microbial activity proceed in frozen soils.txt\n",
      "Bruun.fm.txt\n",
      "Cacao Cultivation under Diverse Shade Tree Cover Allows High Carbon Storage and Sequestration without Yield Losses.txt\n",
      "Can arbuscular mycorrhizal fungi speed up carbon sequestration by enhanced weathering_.txt\n",
      "Can microplastics mediate soil properties, plant growth and carbon_nitrogen turnover in the terrestrial ecosystem_.txt\n",
      "Can molecular genetic techniques improve our understanding of the role the microbial biomass plays in soil aggregation_.txt\n",
      "Carbon acquisition and metabolism changes during fungal biotrophic plant pathogenesis_ insights from.txt\n",
      "Carbon Catabolite Repression in Filamentous Fungi.txt\n",
      "Carbon Farming_ Prospects and Challenges.txt\n",
      "Carbon monoxide emission rates from roasted whole bean and ground coffee.txt\n",
      "Carbon sequestration and greenhouse gas emissions for different rice cultivation practices.txt\n",
      "Carbon sequestration and soil carbon pools in a rice–wheat cropping system_ Effect of long-term use of inorganic fertilizers and organic manure.txt\n",
      "Carbon sequestration and storage value of coffee forest in Southwestern Ethiopia.txt\n",
      "Carbon sequestration by Miscanthus energy crops plantations in a broad range semi-arid marginal land in China.txt\n",
      "Carbon sequestration is related to mycorrhizal fungal community shifts during long‐term succession in boreal forests.txt\n",
      "Carbon sequestration pathway of inorganic carbon in partial nitrification sludge.txt\n",
      "Carbon sequestration potential of sustainable agricultural practices to mitigate climate change in Indian agriculture_ A meta-analysis.txt\n",
      "Carbon sequestration.txt\n",
      "Carbon turnover times shape topsoil carbon difference between Tibetan Plateau and Arctic tundra.txt\n",
      "Carbon, Nitrogen and Water Footprints of Organic Rice and Conventional Rice Production over 4 Years of Cultivation_A Case Study in the Lower North of Thailand.txt\n",
      "Changes in microbial utilization and fate of soil carbon following the addition of different fractions of anaerobic digestate to soils.txt\n",
      "Changes in plant inputs alter soil carbon and microbial communities in forest ecosystems.txt\n",
      "Chapter 3 - Miscanthus_ A Promising Biomass Crop.txt\n",
      "Characterizing the “fungal shunt”_ Parasitic fungi on diatoms affect carbon flow and bacterial communities in aquatic microbial food webs.txt\n",
      "Chemical Composition of Plant Residues Regulates Soil Organic Carbon Turnover in Typical Soils with Contrasting Textures in Northeast China Plain.txt\n",
      "Climate warming alters subsoil but not topsoil carbon dynamics in alpine grassland.txt\n",
      "Comparative Metagenomics of the Active Layer and Permafrost from Low-Carbon Soil in the Canadian High Arctic.txt\n",
      "Comparison of Two Methods for Total Inorganic Carbon Estimation in Three Soil Types in Mediterranean Area.txt\n",
      "Contrasting pathways of carbon sequestration in paddy and upland soils.txt\n",
      "Contribution of soil inorganic carbon to atmospheric CO2_ More important than previously thought.txt\n",
      "Country-level potential of carbon sequestration and environmental benefits by utilizing crop residues for biochar implementation.txt\n",
      "Crop Residue Return Rather Than Organic Manure Increases Soil Aggregate Stability under Corn–Soybean Rotation in Surface Mollisols.txt\n",
      "Crop traits drive soil carbon sequestration under organic farming.txt\n",
      "Crops for Carbon Farming.txt\n",
      "Current Challenges and Pitfalls in Soil Metagenomics.txt\n",
      "Current controversies on mechanisms controlling soil carbon storage_ implications for interactions with practitioners and policy-makers. A review.txt\n",
      "Current views on the regulation of autotrophic carbon dioxide fixation via the Calvin cycle in bacteria.txt\n",
      "Dark microbial CO2 fixation in temperate forest soils increases with CO2 concentration.txt\n",
      "Decay by ectomycorrhizal fungi couples soil organic matter to nitrogen availability.txt\n",
      "December  2017 Issue_AL Aquino et al.pdf.txt\n",
      "Decipher soil organic carbon dynamics and driving forces across China using machine learning.txt\n",
      "Deciphering microbial mechanisms underlying soil organic carbon storage in a wheat-maize rotation system.txt\n",
      "Deep-C storage_ Biological, chemical and physical strategies to enhance carbon stocks in agricultural subsoils.txt\n",
      "Deep-rooted perennial crops differ in capacity to stabilize C inputs in deep soil layers.txt\n",
      "Designing an ecofriendly and carbon-cum-energy efficient production system for the diverse agroecosystem of South Asia.txt\n",
      "doi_10.1016_j.still.2004.09.013.txt\n",
      "doi_10.1016_j.still.2004.09.017.txt\n",
      "doi_10.1016_S0167-1987(03)00121-1.txt\n",
      "doi_10.1098_rstb.2007.2185.txt\n",
      "Dramatic loss of inorganic carbon by nitrogen‐induced soil acidification in Chinese croplands.txt\n",
      "Dynamic Changes and Regional Differences of Net Carbon Sequestration of Food Crops in the Yangtze River Economic Belt of China.txt\n",
      "Dynamics of photosynthetic responses in 10 rubber tree (Hevea brasiliensis) clones in Colombian Amazon_ Implications for breeding strategies.txt\n",
      "Dynamics of soil organic and inorganic carbon in the cropland of upper Yellow River Delta, China.txt\n",
      "Dynamics of Soil Organic Carbon and CO2 Flux under Cover Crop and No-Till Management in Soybean Cropping Systems of the Mid-South (USA).txt\n",
      "Ectomycorrhizal fungi slow soil carbon cycling.txt\n",
      "Effect of long-term tillage and cropping system on portion of fungal and bacterial necromass carbon in soil organic carbon.txt\n",
      "Effect of organic and inorganic fertilizers and rice straw on carbon sequestration and soil fertilit.txt\n",
      "Effect of Organic Carbon and Nitrogen on the Interactions of Morchella spp. and Bacteria Dispersing on Their Mycelium.txt\n",
      "Effect of Time since Afforestation on Soil Organic Carbon Stock and Turnover Rate.txt\n",
      "Effects of Biochar With Inorganic and Organic Fertilizers on Agronomic Traits and Nutrient Absorption of Soybean and Fertility and Microbes in Purple Soil.txt\n",
      "Effects of Biofuel Crop Switchgrass (Panicum virgatum) Cultivation on Soil Carbon Sequestration and Greenhouse Gas Emissions_ A Review.txt\n",
      "Effects of Elevated Carbon Dioxide on Photosynthesis and Carbon Partitioning_ A Perspective on Root Sugar Sensing and Hormonal Crosstalk.txt\n",
      "Effects of exogenous organic_inorganic nitrogen addition on carbon pool distribution and transformation in grassland soil.txt\n",
      "Effects of Fungi on Soil Organic Carbon and Soil Enzyme Activity under Agricultural and Pasture Land of Eastern Türkiye.txt\n",
      "Effects of high CO2 levels on dynamic photosynthesis_ carbon gain, mechanisms, and environmental interactions.txt\n",
      "Effects of integrated soil and water conservation measures on soil aggregate stability, soil organic.txt\n",
      "Effects of long‐term straw return on soil organic carbon storage and sequestration rate in North China upland crops_ A meta‐analysis.txt\n",
      "Effects of reduced tillage, crop residue management and manure application practices on crop yields .txt\n",
      "Effects of rice straw and nitrogen fertilization on greenhouse gas emissions and carbon storage in tropical flooded soil planted with rice.txt\n",
      "Efficacy, Energy Budgeting, and Carbon Footprints of Weed Management in Blackgram (Vigna mungo L.).txt\n",
      "Efforts towards overcoming drought stress in crops_ Revisiting the mechanisms employed by plant growth-promoting bacteria.txt\n",
      "Elevated CO2 and warming cause interactive effects on soil carbon and shifts in carbon use by bacteria.txt\n",
      "Empirical estimates of regional carbon budgets imply reduced global soil heterotrophic respiration.txt\n",
      "Energy budget and carbon footprint in a wheat and maize system under ridge furrow strategy in dry semi humid areas.txt\n",
      "Energy budgeting and carbon footprint of pearl millet - mustard cropping system under conventional and conservation agriculture in rainfed semi-arid agro-ecosystem.txt\n",
      "Enriched CO2 and Root-Associated Fungi (Mycorrhizae) Yield Inverse Effects on Plant Mass and Root Morphology in Six Asclepias Species.txt\n",
      "Environmental Factors Indirectly Impact the Nematode Carbon Budget of Subalpine Spruce Forests.txt\n",
      "Evidence for the primacy of living root inputs, not root or shoot litter, in forming soil organic carbon.txt\n",
      "Exploring the role of ectomycorrhizal fungi in soil carbon dynamics.txt\n",
      "Exposures and Emissions in Coffee Roasting Facilities and Cafés_ Diacetyl, 2,3-Pentanedione, and Other Volatile Organic Compounds.txt\n",
      "Extreme environments_ microbiology leading to specialized metabolites.txt\n",
      "Fast-decaying plant litter enhances soil carbon in temperate forests but not through microbial physiological traits.txt\n",
      "Fekete et al__RGleb_04_2022.indd.txt\n",
      "fenvs-2021-724702 1..13.txt\n",
      "fenvs-2021-756378 1..18.txt\n",
      "Fertilization and Soil Microbial Community_ A Review.txt\n",
      "Fixation of carbon dioxide by chemoautotrophic bacteria in grassland soil under dark conditions.txt\n",
      "Forest carbon sequestration in China and its benefits.txt\n",
      "Forest microbiome and global change.txt\n",
      "From energy to (soil organic) matter.txt\n",
      "Functional Endophytes Regulating Plant Secondary Metabolism_ Current Status, Prospects and Applications.txt\n",
      "Fungal Carbon Sequestration.txt\n",
      "Fungal Endophytes to Combat Biotic and Abiotic Stresses for Climate-Smart and Sustainable Agriculture.txt\n",
      "Fungi That Promote Plant Growth in the Rhizosphere Boost Crop Growth.txt\n",
      "Genome-centric view of carbon processing in thawing permafrost.txt\n",
      "Genome-Resolved Metagenomics Informs the Functional Ecology of Uncultured Acidobacteria in Redox Oscillated Sphagnum Peat.txt\n",
      "Global Environmental Change_ The Causes and Consequences of Disruption to Biogeochemical Cycles.txt\n",
      "Global soil organic carbon–climate interactions_ Why scales matter.txt\n",
      "Global stocks and capacity of mineral-associated soil organic carbon.txt\n",
      "Global subsoil organic carbon turnover times dominantly controlled by soil properties rather than climate.txt\n",
      "Global warming has increased global economic inequality.txt\n",
      "Glycoproteins of arbuscular mycorrhiza for soil carbon sequestration_ Review of mechanisms and controls.txt\n",
      "Gram-positive bacteria control the rapid anabolism of protein-sized soil organic nitrogen compounds questioning the present paradigm.txt\n",
      "Grassland soil carbon sequestration_ Current understanding, challenges, and solutions.txt\n",
      "Grazing-induced microbiome alterations drive soil organic carbon turnover and productivity in meadow steppe.txt\n",
      "Harnessing fungi to mitigate CH4 in natural and engineered systems.txt\n",
      "Hyphae move matter and microbes to mineral microsites_ Integrating the hyphosphere into conceptual models of soil organic matter stabilization.txt\n",
      "IJAgriSci - JANUARY-2023.indd.txt\n",
      "Impact of Silicon on Plant Nutrition and Significance of Silicon Mobilizing Bacteria in Agronomic Practices.txt\n",
      "Impact of twenty pesticides on soil carbon microbial functions and community composition.txt\n",
      "Impacts and mechanisms of biochar on soil microorganisms.txt\n",
      "Impacts of global change on the phyllosphere microbiome.txt\n",
      "Increased microbial growth, biomass, and turnover drive soil organic carbon accumulation at higher plant diversity.txt\n",
      "Increasing contribution of microbial residues to soil organic carbon in grassland restoration chronosequence.txt\n",
      "Influence of cyanobacterial inoculants, elevated carbon dioxide, and temperature on plant and soil nitrogen in soybean.txt\n",
      "Influence of trees and associated variables on soil organic carbon_ a review.txt\n",
      "Inorganic carbon fixation by chemosynthetic ectosymbionts and nutritional transfers to the hydrothermal vent host-shrimp Rimicaris exoculata.txt\n",
      "Integrated Nutrient Management for Rice Yield, Soil Fertility, and Carbon Sequestration.txt\n",
      "Integrating the evidence for a terrestrial carbon sink caused by increasing atmospheric CO2.txt\n",
      "Integrating transcriptomics and metabolomics to analyze quinoa (Chenopodium quinoa Willd.) responses to drought stress and rewatering.txt\n",
      "Interaction between bacterial endophytes and host plants.txt\n",
      "Investigate the role of technology innovation and renewable energy in reducing transport sector CO2 emission in China_ A path toward sustainable development.txt\n",
      "Land and water conservation technologies for building carbon positive villages in India.txt\n",
      "Land use driven change in soil pH affects microbial carbon cycling processes.txt\n",
      "Large‐scale importance of microbial carbon use efficiency and necromass to soil organic carbon.txt\n",
      "Life and death in the soil microbiome_ how ecological processes influence biogeochemistry.txt\n",
      "Life cycle assessment synthesis of the carbon footprint of Arabica coffee_ Case study of Brazil and Vietnam conventional and sustainable coffee production and export to the United Kingdom.txt\n",
      "Life in Dry Soils_ Effects of Drought on Soil Microbial Communities and Processes.txt\n",
      "Linking soil microbial community traits and organic carbon accumulation rate under long-term conservation tillage practices.txt\n",
      "Local temperature increases reduce soil microbial residues and carbon stocks.txt\n",
      "Long-term C and N sequestration under no-till is governed by biomass production of cover crops rather than differences in grass vs. legume biomass quality.txt\n",
      "Long-term integrated soil-crop management improves soil microbial community structure to reduce GHG emission and increase yield.txt\n",
      "Long‐term irrigation reduces soil carbon sequestration by affecting soil microbial communities in agricultural ecosystems of northern China.txt\n",
      "Machine learning and statistical analysis for biomass torrefaction_ A review.txt\n",
      "Machine learning for hydrothermal treatment of biomass_ A review.txt\n",
      "Management of Rhizosphere Microbiota and Plant Production under Drought Stress_ A Comprehensive Review.txt\n",
      "Marginal lands for bioenergy in China; an outlook in status, potential and management.txt\n",
      "Measuring soil organic carbon in croplands at regional scale using airborne imaging spectroscopy.txt\n",
      "Mechanisms of soil organic carbon stability and its response to no‐till_ A global synthesis and perspective.txt\n",
      "Metabolomics of plant root exudates_ From sample preparation to data analysis.txt\n",
      "Metagenomic Analyses of Plant Growth-Promoting and Carbon-Cycling Genes in Maize Rhizosphere Soils with Distinct Land-Use and Management Histories.txt\n",
      "Metagenomics reveals N-induced changes in carbon-degrading genes and microbial communities of tea (Camellia sinensis L.) plantation soil under long-term fertilization.txt\n",
      "Microbial activity in forest soil reflects the changes in ecosystem properties between summer and winter.txt\n",
      "Microbial communities and their genetic repertoire mediate the decomposition of soil organic carbon pools in revegetation shrublands in a desert in northern China.txt\n",
      "Microbial control over carbon cycling in soil.txt\n",
      "Microbial diversity and biogeography in Arctic soils.txt\n",
      "Microbial drought resistance may destabilize soil carbon.txt\n",
      "Microbial formation of stable soil carbon is more efficient from belowground than aboveground input.txt\n",
      "Microbial metabolic response to winter warming stabilizes soil carbon.txt\n",
      "Microbial modulators of soil carbon storage_ integrating genomic and metabolic knowledge for global prediction.txt\n",
      "Microbial spatial footprint as a driver of soil carbon stabilization.txt\n",
      "Microbiomes in agroecosystem_ Diversity, function and assembly mechanisms.txt\n",
      "Microhabitat drive microbial anabolism to promote carbon sequestration during composting.txt\n",
      "Microorganisms in Plant Growth and Development_ Roles in Abiotic Stress Tolerance and Secondary Metabolites Secretion.txt\n",
      "Microsoft Word - IJTM700101 CHIANG_visio.txt\n",
      "Microsoft Word - McCarty et al PDF.doc.txt\n",
      "Modeling Microbial Adaptations to Nutrient Limitation During Litter Decomposition.txt\n",
      "Modeling the Carbon Sequestration Potential of Multifunctional Agroforestry-Based Phytoremediation (MAP) Systems in Chinandega, Nicaragua.txt\n",
      "Multi-scale geospatial agroecosystem modeling_ A case study on the influence of soil data resolution on carbon budget estimates.txt\n",
      "Mycorrhiza and Biochar Effects on Carbon Sequestration.txt\n",
      "Mycorrhizal Fungi as Mediators of Soil Organic Matter Dynamics.txt\n",
      "Mycorrhizal fungi have a potential role in soil carbon storage under elevated CO2 and nitrogen deposition.txt\n",
      "Net photosynthetic CO2 assimilation_ more than just CO2 and O2 reduction cycles.txt\n",
      "Nitrogen loading enhances phosphorus limitation in terrestrial ecosystems with implications for soil carbon cycling.txt\n",
      "Nutrients in the rhizosphere_ A meta-analysis of content, availability, and influencing factors.txt\n",
      "Optimal straw management co‐benefits crop yield and soil carbon sequestration of intensive farming systems.txt\n",
      "Optimizing Inorganic Carbon Sequestration and Crop Yield With Wollastonite Soil Amendment in a Microplot Study.txt\n",
      "Organic carbon accumulation and aggregate formation in soils under organic and inorganic fertilizer management practices in a rice–wheat cropping system.txt\n",
      "Organic carbon transformations in high-Arctic peat soils_ key functions and microorganisms.txt\n",
      "Paired carbon and nitrogen metabolism by ammonia‐oxidizing bacteria and archaea in temperate forest soils.txt\n",
      "Particulate organic matter as a functional soil component for persistent soil organic carbon.txt\n",
      "Pathways of mineral‐associated soil organic matter formation_ Integrating the role of plant carbon source, chemistry, and point of entry.txt\n",
      "Perennials as Future Grain Crops_ Opportunities and Challenges.txt\n",
      "Persistence of dissolved organic matter explained by molecular changes during its passage through soil.txt\n",
      "Persistence of soil organic carbon caused by functional complexity.txt\n",
      "Persistent soil carbon enhanced in Mollisols by well-managed grasslands but not annual grain or dairy forage cropping systems.txt\n",
      "Personal Carbon Budgets_ A Pestle Review.txt\n",
      "Photosynthetic limits on carbon sequestration in croplands.txt\n",
      "Phytolith carbon sequestration in global terrestrial biomes.txt\n",
      "Plant Growth-Promoting Halobacteria and Their Ability to Protect Crops from Abiotic Stress_ An Eco-Friendly Alternative for Saline Soils.txt\n",
      "Plant Growth-Promoting Rhizobacteria (PGPR)_ A Rampart against the Adverse Effects of Drought Stress.txt\n",
      "Plant Nutrition under Climate Change and Soil Carbon Sequestration.txt\n",
      "Plant–microbiome interactions_ from community assembly to plant health.txt\n",
      "pone.0112609 1..9.txt\n",
      "Potential response of dark carbon fixation to global warming in estuarine and coastal waters.txt\n",
      "Potential responses of soil organic carbon to global environmental change.txt\n",
      "pp110000949p.txt\n",
      "pp139162 2104..2115.txt\n",
      "Prediction of soil organic and inorganic carbon contents at a national scale (France) using midinfrared reflectance spectroscopy (MIRS).txt\n",
      "Preliminary investigation of soil organic carbon distribution and turnover patterns, and potential pollution sources in and around a typical coking plant in North China.txt\n",
      "Probabilistic Assessment of Cereal Rye Cover Crop Impacts on Regional Crop Yield and Soil Carbon.txt\n",
      "Production of Biochar and Its Potential Application in Cementitious Composites.txt\n",
      "Progress on the Effect of Nitrogen on Transformation of Soil Organic Carbon.txt\n",
      "Promotion of Carbon Dioxide Biofixation through Metabolic and Enzyme Engineering.txt\n",
      "Protein synthesis increases with photosynthesis via the stimulation of translation initiation.txt\n",
      "Proteomic analysis revealed modulations of carbon and nitrogen by arbuscular mycorrhizal fungi associated with the halophyte Suaeda salsa in a moderately saline environment.txt\n",
      "Quantification of organic carbon pools for Austria’s agricultural soils using a soil information system.txt\n",
      "Quantifying soil carbon stocks and humification through spectroscopic methods_ A scoping assessment in EMBU-Kenya.txt\n",
      "Quantitative assessment of microbial necromass contribution to soil organic matter.txt\n",
      "Rapid Response of Nitrogen Cycling Gene Transcription to Labile Carbon Amendments in a Soil Microbial Community.txt\n",
      "Ratoon rice with direct seeding improves soil carbon sequestration in rice fields and increases grain quality.txt\n",
      "Reduced carbon sequestration potential of biochar in acidic soil.txt\n",
      "Reducing Carbon Footprint of Agriculture—Can Organic Farming Help to Mitigate Climate Change_.txt\n",
      "Regenerative Agriculture—A Literature Review on the Practices and Mechanisms Used to Improve Soil Health.txt\n",
      "Relating microbial community structure to functioning in forest soil organic carbon transformation and turnover.txt\n",
      "Research on the Impact of Agricultural Financial Support on Agricultural Carbon Compensation Rate.txt\n",
      "Response of bacterial community composition and co-occurrence network to straw and straw biochar incorporation.txt\n",
      "Rhizosphere bacteriome structure and functions.txt\n",
      "Rhizospheric microorganisms_ The gateway to a sustainable plant health.txt\n",
      "RNA-Seq Provides New Insights into the Gene Expression Changes in Azoarcus olearius BH72 under Nitrogen-Deficient and Replete Conditions beyond the Nitrogen Fixation Process.txt\n",
      "Role of Arbuscular Mycorrhizal Fungi in Plant Growth Regulation_ Implications in Abiotic Stress Tolerance.txt\n",
      "Role of Fungi in the Biomineralization of Calcite.txt\n",
      "Role of phytolith occluded carbon of crop plants for enhancing soil carbon sequestration in agro-ecosystems.txt\n",
      "Root effects on soil organic carbon_ a double‐edged sword.txt\n",
      "Rooting vegetation models in realism.txt\n",
      "S0021859623000229jra 1..5.txt\n",
      "Seasonal Variations in Grain Yield, Greenhouse Gas Emissions and Carbon Sequestration for Maize Cultivation in Bangladesh.txt\n",
      "Seasonality buffers carbon budget variability across heterogeneous landscapes in Alaskan Arctic tundra.txt\n",
      "SEL-00052-CL 241..254.txt\n",
      "Short-term elevated temperature and CO2 promote photosynthetic induction in the C3 plant Glycine max, but not in the C4 plant Amaranthus tricolor.txt\n",
      "Short-Term Response of Soil Bacteria to Carbon Enrichment in Different Soil Microsites.txt\n",
      "Significant loss of soil inorganic carbon at the continental scale.txt\n",
      "Silicon and soil microorganisms improve rhizospheric soil health with bacterial community, plant gro.txt\n",
      "Sinks for plant surplus carbon explain several ecological phenomena.txt\n",
      "sjss-2022-10427 1..13.txt\n",
      "SM-JBJJ210243 1..17.txt\n",
      "Soil and vegetation carbon turnover times from tropical to boreal forests.txt\n",
      "Soil Autotrophic Bacterial Community Structure and Carbon Utilization Are Regulated by Soil Disturbance—The Case of a 19-Year Field Study.txt\n",
      "Soil Bacteria Mediate Soil Organic Carbon Sequestration under Different Tillage and Straw Management in Rice-Wheat Cropping Systems.txt\n",
      "Soil carbon sequestration - An interplay between soil microbial community and soil organic matter dynamics.txt\n",
      "Soil carbon sequestration by root exudates.txt\n",
      "Soil carbon sequestration, greenhouse gas emissions, and water pollution under different tillage practices.txt\n",
      "Soil carbon storage informed by particulate and mineral-associated organic matter.txt\n",
      "Soil health and carbon management.txt\n",
      "Soil Inorganic Carbon as a Potential Sink in Carbon Storage in Dryland Soils—A Review.txt\n",
      "Soil microbial biomass and bacterial diversity in southern European regions vulnerable to desertification.txt\n",
      "Soil microbial CO2 fixation plays a significant role in terrestrial carbon sink in a dryland ecosystem_ A four-year small-scale field-plot observation on the Tibetan Plateau.txt\n",
      "Soil Microbiome_ A Treasure Trove for Soil Health Sustainability under Changing Climate.txt\n",
      "Soil Moisture Alters the Response of Soil Organic Carbon Mineralization to Litter Addition.txt\n",
      "Soil mycobiome in sustainable agriculture.txt\n",
      "Soil Organic Carbon Development and Turnover in Natural and Disturbed Salt Marsh Environments.txt\n",
      "Soil organic carbon sequestration as influenced by long-term manuring and fertilization in the rice-.txt\n",
      "Soil organic carbon sequestration calculated from depth distribution.txt\n",
      "Soil organic carbon sequestration in croplands can make remarkable contributions to China's carbon neutrality.txt\n",
      "Soil organic carbon stability under natural and anthropogenic-induced perturbations.txt\n",
      "Soil organic carbon turnover recovers faster than plant diversity in the grassland when high nitrogen addition is ceased_ Derived from soil 14C evidences.txt\n",
      "Soil organic matter turnover depending on land use change_ Coupling C_N ratios, δ13C, and lignin biomarkers.txt\n",
      "Soil pH and temperature regulate assembly processes of abundant and rare bacterial communities in agricultural ecosystems.txt\n",
      "Soil-carbon sequestration and soil-carbon fractions, comparison between poplar plantations and corn crops in south-eastern Spain.txt\n",
      "Status of the bioenergy crop miscanthus as a potential reservoir for aphid pests.txt\n",
      "Subsoil Arbuscular Mycorrhizal Fungi for Sustainability and Climate-Smart Agriculture_ A Solution Right Under Our Feet_.txt\n",
      "Systems Biology in Fungal Research.txt\n",
      "Tapping into Plant–Microbiome Interactions through the Lens of Multi-Omics Techniques.txt\n",
      "The Critical Role of Arbuscular Mycorrhizal Fungi to Improve Drought Tolerance and Nitrogen Use Efficiency in Crops.txt\n",
      "The economic value of biochar in crop production and carbon sequestration.txt\n",
      "The effect of concurrent elevation in CO2 and temperature on the growth, photosynthesis, and yield of potato crops.txt\n",
      "The Impact of Drought Stress on Soil Microbial Community, Enzyme Activities and Plants.txt\n",
      "The impact of environmental factors on carbon dioxide fixation by microalgae.txt\n",
      "The Importance of Microorganisms for Sustainable Agriculture—A Review.txt\n",
      "The life of soils_ Integrating the who and how of multifunctionality.txt\n",
      "The microbiota of the grapevine holobiont_ A key component of plant health.txt\n",
      "The rhizosphere microbial complex in plant health_ A review of interaction dynamics.txt\n",
      "The role of plant&#x02010;associated rhizobacteria in plant growth, biocontrol and abiotic stress management.txt\n",
      "The Role of Soil Carbon Sequestration as a Climate Change Mitigation Strategy_ An Australian Case Study.txt\n",
      "The root microbiome_ Community assembly and its contributions to plant fitness.txt\n",
      "The turnover of organic carbon in subsoils. Part 2. Modelling carbon turnover.txt\n",
      "Three-stage carbon release model during macrophyte decomposition.txt\n",
      "Total and dissolved soil organic and inorganic carbon and their relationships in typical loess cropland of Fengu Basin.txt\n",
      "Toward Carbon-Neutral Concrete through Biochar–Cement–Calcium Carbonate Composites_ A Critical Review.txt\n",
      "Toward more sustainable tropical agriculture with cover crops_ Soil microbiome responses to nitrogen management.txt\n",
      "Towards accurate measurements of soil organic carbon stock change in agroecosystems.txt\n",
      "TROPICAL DEFORESTATION AND THE GLOBAL CARBON BUDGET.txt\n",
      "Understanding the joint impacts of soil architecture and microbial dynamics on soil functions_ Insights derived from microscale models.txt\n",
      "Unearthing soil-plant-microbiota crosstalk_ Looking back to move forward.txt\n",
      "Unravelling mechanisms and impacts of day respiration in plant leaves_ an introduction to a Virtual Issue.txt\n",
      "Untitled-1.txt\n",
      "untitled.txt\n",
      "Using soil library hyperspectral reflectance and machine learning to predict soil organic carbon_ Assessing potential of airborne and spaceborne optical soil sensing.txt\n",
      "Vegetation restoration facilitates belowground microbial network complexity and recalcitrant soil organic carbon storage in southwest China karst region.txt\n",
      "Waste Management for Green Concrete Solutions_ A Concise Critical Review.txt\n",
      "Will fungi solve the carbon dilemma_.txt\n",
      "Winter crop rotation intensification to increase rice yield, soil carbon, and microbial diversity.txt\n",
      "Yield sustainability, soil organic carbon sequestration and nutrients balance under long-term combined application of manure and inorganic fertilizers in acidic paddy soil.txt\n",
      "_4D6963726F736F667420576F7264202D20A87EAFF3A569B0B5B4C0A54EAFE0B7BD2E646F63_.txt\n",
      "``Omics'' Technologies for the Study of Soil Carbon Stabilization_ A Review.txt\n"
     ]
    }
   ],
   "source": [
    "# generate and format data files for tensorboard visualisation\n",
    "# using model.save_word2vec format\n",
    "# this step provides (1x vectors, 1x metadata)\n",
    "# doc_tensor--向量 positional data in vector space\n",
    "# metadata -- PDFs\n",
    "# 需要給class視覺化\n",
    "\n",
    "os.chdir(modelDataDir)\n",
    "model.save_word2vec_format('doc_tensor.w2v', doctag_vec=True, word_vec=False)\n",
    "%run ../scripts/word2vec2tensor.py -i doc_tensor.w2v -o pdf_plot\n",
    "\n",
    "text = \"\"    \n",
    "with open('pdf_plot_metadata.tsv', 'w', encoding='UTF-8') as file:\n",
    "    file.write('title\\n')\n",
    "    for fname in os.listdir(txtExtractDir):\n",
    "        if fname.endswith('.txt'):\n",
    "            text = fname\n",
    "            text = re.sub('\\[\\'', '', text)\n",
    "            text = re.sub('\\'\\].txt', '', text)\n",
    "            text = re.sub('\\[', '', text)\n",
    "            text = re.sub('\\].txt', '', text)     \n",
    "            print(text)\n",
    "            file.write(\"%s\\n\" % text)\n",
    "        else:\n",
    "            continue\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "958a3c7c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"methodology\" appears this many times in corpus:\n",
      "{189}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# word occurence check\n",
    "checkWord = \"methodology\"\n",
    "print(\"\\\"\" + str(checkWord) + \"\\\"\" + \" appears this many times in corpus:\") \n",
    "print({model.wv.get_vecattr(checkWord, 'count')})\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b802f56",
   "metadata": {},
   "outputs": [],
   "source": [
    "# assessing the model\n",
    "print(\"assessing the model (this can take a while)\")\n",
    "ranks = []\n",
    "secondRanks = []\n",
    "for doc_id in range(len(model.dv)):\n",
    "        inferredVector = model.infer_vector(trainCorpus[doc_id].words)\n",
    "        sims = model.dv.most_similar([inferredVector], topn=len(model.dv))\n",
    "        rank = [docid for docid, sim in sims].index(doc_id)\n",
    "        ranks.append(rank)\n",
    "        secondRanks.append(sims[1])\n",
    "\n",
    "counter = collections.Counter(ranks)\n",
    "print(bcolors.OKGREEN + \"model assessed\" + bcolors.ENDC)\n",
    "print()\n",
    "print(bcolors.OKBLUE + \"model trained and assessed successfully\" + bcolors.ENDC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7eabd240",
   "metadata": {},
   "outputs": [],
   "source": [
    "###### PART 2 - CHECK SIMILARITY BETWEEN CORPUS AND INCOMING DOCUMENT ######"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50cfb5c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import new document\n",
    "\n",
    "print('importing latin pdf')\n",
    "with smart_open.open(testsDir + '/similarity-test.txt', 'w') as test:\n",
    "    text = pdfminer_extraction(testsDir + '/german.pdf')\n",
    "    text = \"\".join(line.strip(\"\\n\") for line in text) \n",
    "    test.write(text)\n",
    "    \n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "540252ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenize and tag new document\n",
    "print('tag and tokenize pdf for doc2vec processing')\n",
    "def read_text(fname, tokens_only=False):\n",
    "    with smart_open.open(fname, encoding=\"iso-8859-1\") as f:\n",
    "        for i, line in enumerate(f):\n",
    "            tokens = gensim.utils.simple_preprocess(line)\n",
    "            if tokens_only:\n",
    "                yield tokens\n",
    "            else:\n",
    "                # For training data, add tags\n",
    "                yield gensim.models.doc2vec.TaggedDocument(tokens, [i])\n",
    "                \n",
    "similarityTest = list(read_text(testsDir + '/similarity-test.txt'))\n",
    "\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53e40139",
   "metadata": {},
   "outputs": [],
   "source": [
    "# check for similarity against the entire corpus\n",
    "print('compare similarity between incoming pdf and text corpus (the model)')\n",
    "novel_vector = model.infer_vector(similarityTest[0].words)\n",
    "similarity_scores = model.dv.most_similar([novel_vector])\n",
    "average = 0\n",
    "for score in similarity_scores:\n",
    "    average += score[1]\n",
    "overall_similarity = average/len(similarity_scores)\n",
    "\n",
    "print(\"> Incoming pdf is \" + format((overall_similarity - 1) * -1, '.0%') + \" similar to corpus\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1eee1e0c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}