jieba_learing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "该文档对 jieba 分词器 进行一个简单的学习"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function, unicode_literals\n",
    "import sys\n",
    "sys.path.append(\"../\")\n",
    "import jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 载入字典\n",
    "jieba.load_userdict(\"userdict.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 词性标注\n",
    "import jieba.posseg as pseg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 调整字典，可在程序中动态调整 \n",
    "jieba.add_word('石墨烯')\n",
    "jieba.add_word('凱特琳')\n",
    "jieba.del_word('自定义词')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 这是一个字符串\n",
    "test_sent = (\n",
    "\"李小福是创新办主任也是云计算方面的专家; 什么是八一双鹿\\n\"\n",
    "\"例如我输入一个带“韩玉赏鉴”的标题，在自定义词库中也增加了此词为N类\\n\"\n",
    "\"「台中」正確應該不會被切開。mac上可分出「石墨烯」；此時又可以分出來凱特琳了。\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "李小福/是/创新办/主任/也/是/云计算/方面/的/专家/;/ /什么/是/八一双鹿/\n",
      "/例如/我/输入/一个/带/“/韩玉赏鉴/”/的/标题/，/在/自定义词/库中/也/增加/了/此/词为/N/类/\n",
      "/「/台中/」/正確/應該/不會/被/切開/。/mac/上/可/分出/「/石墨烯/」/；/此時/又/可以/分出/來/凱特琳/了/。\n"
     ]
    }
   ],
   "source": [
    "words = jieba.cut(test_sent)\n",
    "print('/'.join(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========================================\n",
      "李小福 / nr ,  是 / v ,  创新办 / i ,  主任 / b ,  也 / d ,  是 / v ,  云计算 / x ,  方面 / n ,  的 / uj ,  专家 / n ,  ; / x ,    / x ,  什么 / r ,  是 / v ,  八一双鹿 / nz ,  \n",
      " / x ,  例如 / v ,  我 / r ,  输入 / v ,  一个 / m ,  带 / v ,  “ / x ,  韩玉赏鉴 / nz ,  ” / x ,  的 / uj ,  标题 / n ,  ， / x ,  在 / p ,  自定义词 / n ,  库中 / nrt ,  也 / d ,  增加 / v ,  了 / ul ,  此 / r ,  词 / n ,  为 / p ,  N / eng ,  类 / q ,  \n",
      " / x ,  「 / x ,  台中 / s ,  」 / x ,  正確 / ad ,  應該 / v ,  不 / d ,  會 / v ,  被 / p ,  切開 / ad ,  。 / x ,  mac / eng ,  上 / f ,  可 / v ,  分出 / v ,  「 / x ,  石墨烯 / x ,  」 / x ,  ； / x ,  此時 / c ,  又 / d ,  可以 / c ,  分出 / v ,  來 / zg ,  凱特琳 / nz ,  了 / ul ,  。 / x ,  \n",
      "========================================\n"
     ]
    }
   ],
   "source": [
    "print(\"=\"*40)\n",
    "result = pseg.cut(test_sent)\n",
    "\n",
    "for w in result:\n",
    "    print(w.word, \"/\", w.flag, \", \", end=' ')\n",
    "print(\"\\n\" + \"=\"*40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "easy_install/ /is/ /great\n"
     ]
    }
   ],
   "source": [
    "terms = jieba.cut('easy_install is great')\n",
    "print('/'.join(terms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "python/ /的/正则表达式/是/好用/的\n"
     ]
    }
   ],
   "source": [
    "terms = jieba.cut('python 的正则表达式是好用的')\n",
    "print('/'.join(terms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "testlist = [\n",
    "('今天天气不错', ('今天', '天气')),\n",
    "('如果放到post中将出错。', ('中', '将')),\n",
    "('我们中出了一个叛徒', ('中', '出')),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========================================\n",
      "今天天气 Before: 5, After: 0\n",
      "中将 Before: 494, After: 494\n",
      "中出 Before: 3, After: 3\n",
      "----------------------------------------\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(\"=\"*40)\n",
    "# test frequency tune\n",
    "testlist = [\n",
    "('今天天气不错', ('今天', '天气')),\n",
    "('如果放到post中将出错。', ('中', '将')),\n",
    "('我们中出了一个叛徒', ('中', '出')),\n",
    "]\n",
    "\n",
    "for sent, seg in testlist:\n",
    "    # 分词了\n",
    "    # print('/'.join(jieba.cut(sent, HMM=False)))\n",
    "    # 把 seg 粘在了一起\n",
    "    word = ''.join(seg)\n",
    "    \n",
    "    print('%s Before: %s, After: %s' % (word, jieba.get_FREQ(word), jieba.suggest_freq(seg, True)))\n",
    "    #print('/'.join(jieba.cut(sent, HMM=False)))\n",
    "print(\"-\"*40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}