[{"data":1,"prerenderedAt":668},["ShallowReactive",2],{"content-\u002Fcontents\u002Foptunalightgbm":3,"surroundPost-\u002Fcontents\u002Foptunalightgbm":659},{"id":4,"title":5,"body":6,"createdAt":646,"description":647,"draft":648,"extension":649,"meta":650,"navigation":107,"path":651,"seo":652,"stem":653,"tags":654,"thumbnail":657,"updatedAt":646,"__hash__":658},"contents\u002Fcontents\u002Foptunalightgbm.md","Optunaとoptuna-dashboardの使い方をLightgbmを例に学ぶ",{"type":7,"value":8,"toc":640},"minimark",[9,20,23,26,31,34,42,52,59,63,66,75,177,184,187,263,266,269,273,282,285,516,519,525,531,537,541,550,553,559,562,573,593,600,606,609,614,617,620,623,627,630,633,636],[10,11,12,19],"p",{},[13,14,18],"a",{"href":15,"rel":16},"https:\u002F\u002Fgithub.com\u002Foptuna\u002Foptuna",[17],"nofollow","Optuna"," は機械学習で最適なハイパーパラメータを探してくれる Python ライブラリです。",[10,21,22],{},"特徴は自動でハイパーパラメータを探索する、最新のアルゴリズムを使ってる、並列化も容易であることです。",[10,24,25],{},"この記事ははそんな Optuna を 1 から勉強する記事です。",[27,28,30],"h2",{"id":29},"optuna-準備","optuna 準備",[10,32,33],{},"環境は Mac で VScode から Jupyter を使って実験します。",[10,35,36,37,41],{},"poetry を使って環境を準備します。以下が",[38,39,40],"code",{},"pyproject.toml","です。後半で使うものもいろいろ入ってるので、optuna だけなら optuna のみで大丈夫です",[43,44,49],"pre",{"className":45,"code":47,"language":48},[46],"language-text","[tool.poetry]\nname = \"optuna_lightgbm\"\nversion = \"0.1.0\"\ndescription = \"\"\nauthors = [\"Your Name \u003Cuser@example.com>\"]\n\n[tool.poetry.dependencies]\npython = \"^3.8\"\njupyter = \"^1.0.0\"\npandas = \"^1.4.2\"\noptuna = \"^2.10.0\"\nlightgbm = \"^3.3.2\"\nsklearn = \"^0.0\"\noptuna-dashboard = \"^0.6.4\"\n\n[tool.poetry.dev-dependencies]\n\n[build-system]\nrequires = [\"poetry-core>=1.0.0\"]\nbuild-backend = \"poetry.core.masonry.api\"\n\n","text",[38,50,47],{"__ignoreMap":51},"",[10,53,54,55,58],{},"これで",[38,56,57],{},"poetry install","で完了です。",[27,60,62],{"id":61},"optuna-基本的な使い方","optuna 基本的な使い方",[10,64,65],{},"公式サイトのチュートリアルをやってみます。",[10,67,68,69,74],{},"以下が",[13,70,73],{"href":71,"rel":72},"https:\u002F\u002Foptuna.readthedocs.io\u002Fen\u002Fstable\u002Ftutorial\u002F10_key_features\u002F001_first.html#sphx-glr-download-tutorial-10-key-features-001-first-py",[17],"こちら","のチュートリアルのコードです。",[43,76,80],{"className":77,"code":78,"language":79,"meta":51,"style":51},"language-py shiki shiki-themes github-dark","# %%\nimport optuna\nfrom optuna.trial import Trial\n\ndef objective(trial:Trial):\n    x = trial.suggest_float(\"x\", -10, 10)\n    return (x - 2) ** 2\n\n# %%\nstudy = optuna.create_study()\nstudy.optimize(objective, n_trials=100)\n\n# %%\nbest_params = study.best_params\nfound_x = best_params[\"x\"]\nprint(\"Found x: {}, (x - 2)^2: {}\".format(found_x, (found_x - 2) ** 2))\n\n","py",[38,81,82,90,96,102,109,115,121,127,132,137,143,149,154,159,165,171],{"__ignoreMap":51},[83,84,87],"span",{"class":85,"line":86},"line",1,[83,88,89],{},"# %%\n",[83,91,93],{"class":85,"line":92},2,[83,94,95],{},"import optuna\n",[83,97,99],{"class":85,"line":98},3,[83,100,101],{},"from optuna.trial import Trial\n",[83,103,105],{"class":85,"line":104},4,[83,106,108],{"emptyLinePlaceholder":107},true,"\n",[83,110,112],{"class":85,"line":111},5,[83,113,114],{},"def objective(trial:Trial):\n",[83,116,118],{"class":85,"line":117},6,[83,119,120],{},"    x = trial.suggest_float(\"x\", -10, 10)\n",[83,122,124],{"class":85,"line":123},7,[83,125,126],{},"    return (x - 2) ** 2\n",[83,128,130],{"class":85,"line":129},8,[83,131,108],{"emptyLinePlaceholder":107},[83,133,135],{"class":85,"line":134},9,[83,136,89],{},[83,138,140],{"class":85,"line":139},10,[83,141,142],{},"study = optuna.create_study()\n",[83,144,146],{"class":85,"line":145},11,[83,147,148],{},"study.optimize(objective, n_trials=100)\n",[83,150,152],{"class":85,"line":151},12,[83,153,108],{"emptyLinePlaceholder":107},[83,155,157],{"class":85,"line":156},13,[83,158,89],{},[83,160,162],{"class":85,"line":161},14,[83,163,164],{},"best_params = study.best_params\n",[83,166,168],{"class":85,"line":167},15,[83,169,170],{},"found_x = best_params[\"x\"]\n",[83,172,174],{"class":85,"line":173},16,[83,175,176],{},"print(\"Found x: {}, (x - 2)^2: {}\".format(found_x, (found_x - 2) ** 2))\n",[10,178,179,180,183],{},"チュートリアルにはないですが、勉強のために",[38,181,182],{},"objective","関数の引数に型を書いてます。",[10,185,186],{},"上記コードを噛み砕くと、",[188,189,190,222,250],"ol",{},[191,192,193,194],"li",{},"最適化対象関数を作成",[195,196,197,203,208,217],"ul",{},[191,198,199,200,202],{},"最適化したい関数を",[38,201,182],{},"という Python の関数の戻り値に設定",[191,204,205,207],{},[38,206,182],{},"関数の引数に Trial 型の引数を設定",[191,209,210,212,213,216],{},[38,211,182],{},"関数内部で Trial クラスのメソッドで",[38,214,215],{},"suggest_float","を呼び出す",[191,218,219,221],{},[38,220,215],{},"では最適化したい変数名と探索範囲を指定する",[191,223,224,225],{},"最適化パラメータ探索実行",[195,226,227,241],{},[191,228,229,232,233,236,237,240],{},[38,230,231],{},"optuna","の",[38,234,235],{},"create_study()","メソッドで optuna の学習用インスタンス（",[38,238,239],{},"study","）を作成",[191,242,243,245,246,249],{},[38,244,239],{},"から",[38,247,248],{},"optimize","メソッドを呼び出し探索スタート。引数には繰り返し回数を指定",[191,251,252,253],{},"結果の取り出し",[195,254,255],{},[191,256,257,232,259,262],{},[38,258,239],{},[38,260,261],{},"best_params","に（おそらく）探索したパラメータ群が辞書で入ってるので、設定した変数名を指定して取り出す",[10,264,265],{},"上記は単純な凸関数の最小値となる変数を求めるものでしたが、実際に使いたい場面は機械学習アルゴリズムのハイパーパラメーター探索です。",[10,267,268],{},"そこで続いては、機械学習アルゴリズムの LightGBM の example を試してみます。",[27,270,272],{"id":271},"optuna-を-lightgbm-に使う","optuna を LightGBM に使う",[10,274,275,276,281],{},"optuna がそのまんまの ",[13,277,280],{"href":278,"rel":279},"https:\u002F\u002Fgithub.com\u002Foptuna\u002Foptuna-examples\u002Fblob\u002Fmain\u002Flightgbm\u002Flightgbm_simple.py",[17],"example"," を用意してるので活用します。",[10,283,284],{},"丸写しはあれなのでちょっとコードとコメントを追記してます。",[43,286,288],{"className":77,"code":287,"language":79,"meta":51,"style":51},"import numpy as np\nimport optuna\nfrom optuna.trial import Trial\n\nimport lightgbm as lgb\nimport sklearn.datasets\nimport sklearn.metrics\nfrom sklearn.model_selection import train_test_split\n\n# ログが多いのでoptunaのログをいったん非表示に\noptuna.logging.disable_default_handler()\n\ndef objective(trial:Trial):\n    data, target = sklearn.datasets.load_breast_cancer(return_X_y=True)\n    train_x, valid_x, train_y, valid_y = train_test_split(data, target, test_size=0.25)\n    dtrain = lgb.Dataset(train_x, label=train_y)\n\n    # lightgbmのパラメータたち\n    # パラメータの型に応じてtrialのメソッドを変える\n    param = {\n        \"objective\": \"binary\",\n        \"metric\": \"binary_logloss\",\n        \"verbosity\": -1,\n        \"boosting_type\": \"gbdt\",\n        \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 10.0, log=True),\n        \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 10.0, log=True),\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 2, 256),\n        \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.4, 1.0),\n        \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.4, 1.0),\n        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 7),\n        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n    }\n\n    gbm = lgb.train(param, dtrain)\n    preds = gbm.predict(valid_x)\n    pred_labels = np.rint(preds)\n    accuracy = sklearn.metrics.accuracy_score(valid_y, pred_labels)\n    return accuracy\n\n# 最大化する場合は明示的に書く。今回はaccuracyを最大化する最適化問題\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=100)\n",[38,289,290,295,299,303,307,312,317,322,327,331,336,341,345,349,354,359,364,369,375,381,387,393,399,405,411,417,423,429,435,441,447,453,459,464,470,476,482,488,494,499,505,511],{"__ignoreMap":51},[83,291,292],{"class":85,"line":86},[83,293,294],{},"import numpy as np\n",[83,296,297],{"class":85,"line":92},[83,298,95],{},[83,300,301],{"class":85,"line":98},[83,302,101],{},[83,304,305],{"class":85,"line":104},[83,306,108],{"emptyLinePlaceholder":107},[83,308,309],{"class":85,"line":111},[83,310,311],{},"import lightgbm as lgb\n",[83,313,314],{"class":85,"line":117},[83,315,316],{},"import sklearn.datasets\n",[83,318,319],{"class":85,"line":123},[83,320,321],{},"import sklearn.metrics\n",[83,323,324],{"class":85,"line":129},[83,325,326],{},"from sklearn.model_selection import train_test_split\n",[83,328,329],{"class":85,"line":134},[83,330,108],{"emptyLinePlaceholder":107},[83,332,333],{"class":85,"line":139},[83,334,335],{},"# ログが多いのでoptunaのログをいったん非表示に\n",[83,337,338],{"class":85,"line":145},[83,339,340],{},"optuna.logging.disable_default_handler()\n",[83,342,343],{"class":85,"line":151},[83,344,108],{"emptyLinePlaceholder":107},[83,346,347],{"class":85,"line":156},[83,348,114],{},[83,350,351],{"class":85,"line":161},[83,352,353],{},"    data, target = sklearn.datasets.load_breast_cancer(return_X_y=True)\n",[83,355,356],{"class":85,"line":167},[83,357,358],{},"    train_x, valid_x, train_y, valid_y = train_test_split(data, target, test_size=0.25)\n",[83,360,361],{"class":85,"line":173},[83,362,363],{},"    dtrain = lgb.Dataset(train_x, label=train_y)\n",[83,365,367],{"class":85,"line":366},17,[83,368,108],{"emptyLinePlaceholder":107},[83,370,372],{"class":85,"line":371},18,[83,373,374],{},"    # lightgbmのパラメータたち\n",[83,376,378],{"class":85,"line":377},19,[83,379,380],{},"    # パラメータの型に応じてtrialのメソッドを変える\n",[83,382,384],{"class":85,"line":383},20,[83,385,386],{},"    param = {\n",[83,388,390],{"class":85,"line":389},21,[83,391,392],{},"        \"objective\": \"binary\",\n",[83,394,396],{"class":85,"line":395},22,[83,397,398],{},"        \"metric\": \"binary_logloss\",\n",[83,400,402],{"class":85,"line":401},23,[83,403,404],{},"        \"verbosity\": -1,\n",[83,406,408],{"class":85,"line":407},24,[83,409,410],{},"        \"boosting_type\": \"gbdt\",\n",[83,412,414],{"class":85,"line":413},25,[83,415,416],{},"        \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 10.0, log=True),\n",[83,418,420],{"class":85,"line":419},26,[83,421,422],{},"        \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 10.0, log=True),\n",[83,424,426],{"class":85,"line":425},27,[83,427,428],{},"        \"num_leaves\": trial.suggest_int(\"num_leaves\", 2, 256),\n",[83,430,432],{"class":85,"line":431},28,[83,433,434],{},"        \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.4, 1.0),\n",[83,436,438],{"class":85,"line":437},29,[83,439,440],{},"        \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.4, 1.0),\n",[83,442,444],{"class":85,"line":443},30,[83,445,446],{},"        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 7),\n",[83,448,450],{"class":85,"line":449},31,[83,451,452],{},"        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n",[83,454,456],{"class":85,"line":455},32,[83,457,458],{},"    }\n",[83,460,462],{"class":85,"line":461},33,[83,463,108],{"emptyLinePlaceholder":107},[83,465,467],{"class":85,"line":466},34,[83,468,469],{},"    gbm = lgb.train(param, dtrain)\n",[83,471,473],{"class":85,"line":472},35,[83,474,475],{},"    preds = gbm.predict(valid_x)\n",[83,477,479],{"class":85,"line":478},36,[83,480,481],{},"    pred_labels = np.rint(preds)\n",[83,483,485],{"class":85,"line":484},37,[83,486,487],{},"    accuracy = sklearn.metrics.accuracy_score(valid_y, pred_labels)\n",[83,489,491],{"class":85,"line":490},38,[83,492,493],{},"    return accuracy\n",[83,495,497],{"class":85,"line":496},39,[83,498,108],{"emptyLinePlaceholder":107},[83,500,502],{"class":85,"line":501},40,[83,503,504],{},"# 最大化する場合は明示的に書く。今回はaccuracyを最大化する最適化問題\n",[83,506,508],{"class":85,"line":507},41,[83,509,510],{},"study = optuna.create_study(direction=\"maximize\")\n",[83,512,514],{"class":85,"line":513},42,[83,515,148],{},[10,517,518],{},"あれ、思ったより簡単だぞ。",[10,520,521,522,524],{},"結局",[38,523,182],{},"関数に最適化したいパラメータの設定と目的となる値を返り値にするだけか。",[10,526,527,528,530],{},"ポイントは",[38,529,182],{},"関数内でデータセット準備 → 学習 → 評価までやらないといけないこと。",[10,532,533,534,536],{},"逆にいうと、既存コードを丸々",[38,535,182],{},"関数に入れるということもできそう。",[27,538,540],{"id":539},"optuna-dashboard-の使い方","Optuna dashboard の使い方",[10,542,543,544,549],{},"Optuna の最適化結果を可視化する",[13,545,548],{"href":546,"rel":547},"https:\u002F\u002Fgithub.com\u002Foptuna\u002Foptuna-dashboard",[17],"optuna-dashboard","なるものがあるので使ってみました。",[10,551,552],{},"使うには pip などでインストールする必要があります。",[43,554,557],{"className":555,"code":556,"language":48},[46],"poetry add optuna-dashboard\n",[38,558,556],{"__ignoreMap":51},[10,560,561],{},"ドキュメントにはあまり説明がありませんが、実際に可視化するにはまず optuna の最適化結果を sqlite に書き出す必要があります。",[10,563,564,565,568,569,572],{},"やり方は",[38,566,567],{},"optuna.create_study()","の引数に",[38,570,571],{},"study_name=study_name, storage=storage_name","を加えて以下のようにします。",[43,574,576],{"className":77,"code":575,"language":79,"meta":51,"style":51},"study_name = \"example-study\"  # Unique identifier of the study.\nstorage_name = \"sqlite:\u002F\u002F\u002F{}.db\".format(study_name)\noptuna.create_study(study_name=study_name, storage=storage_name)\n",[38,577,578,583,588],{"__ignoreMap":51},[83,579,580],{"class":85,"line":86},[83,581,582],{},"study_name = \"example-study\"  # Unique identifier of the study.\n",[83,584,585],{"class":85,"line":92},[83,586,587],{},"storage_name = \"sqlite:\u002F\u002F\u002F{}.db\".format(study_name)\n",[83,589,590],{"class":85,"line":98},[83,591,592],{},"optuna.create_study(study_name=study_name, storage=storage_name)\n",[10,594,595,596,599],{},"最適化が実行されると",[38,597,598],{},"{storage_name}.db","というファイルが生成されるので、以下コマンドの引数に指定して実行します。",[43,601,604],{"className":602,"code":603,"language":48},[46],"optuna-dashboard sqlite:\u002F\u002F\u002Fexample-study.db\n",[38,605,603],{"__ignoreMap":51},[10,607,608],{},"そうすると以下のようなダッシュボードがブラウザの logalhost:8080 で表示されます。",[610,611],"img",{"alt":612,"img-src":613},"dashboard","\u002Fimg\u002Foptuna\u002Fdashboard.png",[10,615,616],{},"ダッシュボード の内容ですが、正直わからないことが多いです。",[10,618,619],{},"ちょっと調べてみると、重要なハイパーパラメータとパラメータ間の相関関係などが表示されるようです。",[10,621,622],{},"試しに LightGBM のパターンで可視化してみました。",[610,624],{"alt":625,"img-src":626},"dashboard 2","\u002Fimg\u002Foptuna\u002Fdashboard2.png",[10,628,629],{},"なるほど。確かに参考になりそうな結果。",[10,631,632],{},"いずれにせよ LightGBM の各種パラメータの理解は必須になりますね。",[10,634,635],{},"今回は以上です！",[637,638,639],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":51,"searchDepth":92,"depth":92,"links":641},[642,643,644,645],{"id":29,"depth":92,"text":30},{"id":61,"depth":92,"text":62},{"id":271,"depth":92,"text":272},{"id":539,"depth":92,"text":540},"2022-05-18","機械学習のハイパーパラメータの最適化に使用されるoptunaを1から使ってみました。例として単純な凸関数やLightGBMを使用しました。またoptuna-dashboardも試してみました。",false,"md",{},"\u002Fcontents\u002Foptunalightgbm",{"title":5,"description":647},"contents\u002Foptunalightgbm",[655,656],"2022","機械学習","\u002Fimg\u002Ftwitter-card.png","BRpkMZuTo-CdGts7llmORbzcQRJG9-DL55xepK8-FaY",[660,664],{"title":661,"path":662,"stem":663,"children":-1},"OpenAIできることを再確認","\u002Fcontents\u002Fopenai","contents\u002Fopenai",{"title":665,"path":666,"stem":667,"children":-1},"【学び】「アウトプット大全」を読んでわかったこと","\u002Fcontents\u002Foutput_taizen","contents\u002Foutput_taizen",1784936718662]