{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 34 }, "colab_type": "code", "id": "KBkpRgBCBS2_", "outputId": "2ea53dde-323b-4374-ca85-6a27fde311f0" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Using TensorFlow backend.\n" ] } ], "source": [ "#!pip install -q textgenrnn\n", "import os\n", "import pandas as pd\n", "from textgenrnn import textgenrnn" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Data" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Total number of papers: 27188\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
linktimefavoritesrtsauthorscategorypublishedsummarytitletweeted
0arxiv.org/abs/1611.10003NaNNaNNaN[Tom A. F. Anderson, C. -H. Ruan]q-bio.NC2016-11-30 05:17:11In summary of the research findings presented ...Vocabulary and the Brain: Evidence from Neuroi...0
1arxiv.org/abs/1611.10007NaNNaNNaN[M. Amin Rahimian, Amir G. Aghdam]cs.SY2016-11-30 05:37:11In this paper, structural controllability of a...Structural Controllability of Multi-Agent Netw...0
2arxiv.org/abs/1611.10010NaNNaNNaN[Debidatta Dwibedi, Tomasz Malisiewicz, Vijay ...cs.CV2016-11-30 06:00:47We present a Deep Cuboid Detector which takes ...Deep Cuboid Detection: Beyond 2D Bounding Boxes0
3arxiv.org/abs/1611.100122016-12-01 01:46:1211.02.0[Jonathan Huang, Vivek Rathod, Chen Sun, Mengl...cs.CV2016-11-30 06:06:15In this paper, we study the trade-off between ...Speed/accuracy trade-offs for modern convoluti...1
4arxiv.org/abs/1611.10014NaNNaNNaN[Yoones Hashemi, Amir H. Banihashemi]cs.IT2016-11-30 06:12:45In this paper, we propose a characterization o...Characterization and Efficient Exhaustive Sear...0
\n", "
" ], "text/plain": [ " link time favorites rts \\\n", "0 arxiv.org/abs/1611.10003 NaN NaN NaN \n", "1 arxiv.org/abs/1611.10007 NaN NaN NaN \n", "2 arxiv.org/abs/1611.10010 NaN NaN NaN \n", "3 arxiv.org/abs/1611.10012 2016-12-01 01:46:12 11.0 2.0 \n", "4 arxiv.org/abs/1611.10014 NaN NaN NaN \n", "\n", " authors category \\\n", "0 [Tom A. F. Anderson, C. -H. Ruan] q-bio.NC \n", "1 [M. Amin Rahimian, Amir G. Aghdam] cs.SY \n", "2 [Debidatta Dwibedi, Tomasz Malisiewicz, Vijay ... cs.CV \n", "3 [Jonathan Huang, Vivek Rathod, Chen Sun, Mengl... cs.CV \n", "4 [Yoones Hashemi, Amir H. Banihashemi] cs.IT \n", "\n", " published summary \\\n", "0 2016-11-30 05:17:11 In summary of the research findings presented ... \n", "1 2016-11-30 05:37:11 In this paper, structural controllability of a... \n", "2 2016-11-30 06:00:47 We present a Deep Cuboid Detector which takes ... \n", "3 2016-11-30 06:06:15 In this paper, we study the trade-off between ... \n", "4 2016-11-30 06:12:45 In this paper, we propose a characterization o... \n", "\n", " title tweeted \n", "0 Vocabulary and the Brain: Evidence from Neuroi... 0 \n", "1 Structural Controllability of Multi-Agent Netw... 0 \n", "2 Deep Cuboid Detection: Beyond 2D Bounding Boxes 0 \n", "3 Speed/accuracy trade-offs for modern convoluti... 1 \n", "4 Characterization and Efficient Exhaustive Sear... 0 " ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "BASE_DIR = os.getcwd()\n", "DATA_DIR = os.path.join(BASE_DIR, '..', 'datasets')\n", "\n", "# from: https://github.com/amauboussin/arxiv-twitterbot\n", "arxiv_df = pd.read_csv(os.path.join(DATA_DIR, 'brundage_bot.csv'))\n", "print('Total number of papers:', len(arxiv_df))\n", "arxiv_df.head()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of deep learning papers: 10003\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
linktimefavoritesrtsauthorscategorypublishedsummarytitletweeted
2arxiv.org/abs/1611.10010NaNNaNNaN[Debidatta Dwibedi, Tomasz Malisiewicz, Vijay ...cs.CV2016-11-30 06:00:47We present a Deep Cuboid Detector which takes ...Deep Cuboid Detection: Beyond 2D Bounding Boxes0
3arxiv.org/abs/1611.100122016-12-01 01:46:1211.02.0[Jonathan Huang, Vivek Rathod, Chen Sun, Mengl...cs.CV2016-11-30 06:06:15In this paper, we study the trade-off between ...Speed/accuracy trade-offs for modern convoluti...1
5arxiv.org/abs/1611.10017NaNNaNNaN[Gou Koutaki, Keiichiro Shirai, Mitsuru Ambai]cs.CV2016-11-30 06:35:39In this paper, we propose a learning-based sup...Fast Supervised Discrete Hashing and its Analysis0
10arxiv.org/abs/1611.10031NaNNaNNaN[Peng Liu, Hui Zhang, Kie B. Eom]cs.LG2016-11-30 07:34:46Active deep learning classification of hypersp...Active Deep Learning for Classification of Hyp...0
11arxiv.org/abs/1611.10038NaNNaNNaN[Si Li, Nianwen Xue]cs.CL2016-11-30 07:53:34A patent is a property right for an invention ...Towards Accurate Word Segmentation for Chinese...0
\n", "
" ], "text/plain": [ " link time favorites rts \\\n", "2 arxiv.org/abs/1611.10010 NaN NaN NaN \n", "3 arxiv.org/abs/1611.10012 2016-12-01 01:46:12 11.0 2.0 \n", "5 arxiv.org/abs/1611.10017 NaN NaN NaN \n", "10 arxiv.org/abs/1611.10031 NaN NaN NaN \n", "11 arxiv.org/abs/1611.10038 NaN NaN NaN \n", "\n", " authors category \\\n", "2 [Debidatta Dwibedi, Tomasz Malisiewicz, Vijay ... cs.CV \n", "3 [Jonathan Huang, Vivek Rathod, Chen Sun, Mengl... cs.CV \n", "5 [Gou Koutaki, Keiichiro Shirai, Mitsuru Ambai] cs.CV \n", "10 [Peng Liu, Hui Zhang, Kie B. Eom] cs.LG \n", "11 [Si Li, Nianwen Xue] cs.CL \n", "\n", " published summary \\\n", "2 2016-11-30 06:00:47 We present a Deep Cuboid Detector which takes ... \n", "3 2016-11-30 06:06:15 In this paper, we study the trade-off between ... \n", "5 2016-11-30 06:35:39 In this paper, we propose a learning-based sup... \n", "10 2016-11-30 07:34:46 Active deep learning classification of hypersp... \n", "11 2016-11-30 07:53:34 A patent is a property right for an invention ... \n", "\n", " title tweeted \n", "2 Deep Cuboid Detection: Beyond 2D Bounding Boxes 0 \n", "3 Speed/accuracy trade-offs for modern convoluti... 1 \n", "5 Fast Supervised Discrete Hashing and its Analysis 0 \n", "10 Active Deep Learning for Classification of Hyp... 0 \n", "11 Towards Accurate Word Segmentation for Chinese... 0 " ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# cs.AI Artificial Intelligence -- cs.CL Computation and Language -- cs.CV Computer Vision and Pattern Recognition\n", "# cs.LG Learning -- cs.NE Neural and Evolutionary Computing -- stat.ML Machine Learning\n", "\n", "arxiv_df = arxiv_df.loc[(arxiv_df.category == 'cs.AI') | (arxiv_df.category == 'cs.CL') |\n", " (arxiv_df.category == 'cs.CV') | (arxiv_df.category == 'cs.LG') |\n", " (arxiv_df.category == 'cs.NE') | (arxiv_df.category == 'stat.ML')]\n", "print('Number of deep learning papers:', len(arxiv_df))\n", "arxiv_df.head()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'Fusion of EEG and Musical Features in Continuous Music-emotion\\n Recognition'" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "arxiv_df.loc[32, 'title']" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'Fusion of EEG and Musical Features in Continuous Music-emotion Recognition'" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# remove newlines in arxiv titles\n", "arxiv_df.title = arxiv_df.title.apply(lambda x: x.replace('\\n ', ''))\n", "\n", "arxiv_df.loc[32, 'title']" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean paper title length: 69.33 chars\n" ] } ], "source": [ "paper_len = (arxiv_df.title.str.len()).mean()\n", "print(f'mean paper title length: {paper_len:.2f} chars')" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Total number of songs: 9850\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
title
0Silver Lining
1Four Winds
2Half Love
3Sowa (Alex Garett & Greg Herma Edit)
4Killing My Time
\n", "
" ], "text/plain": [ " title\n", "0 Silver Lining\n", "1 Four Winds\n", "2 Half Love\n", "3 Sowa (Alex Garett & Greg Herma Edit)\n", "4 Killing My Time" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "songs_df = pd.read_json(os.path.join(DATA_DIR, 'song_titles_5yrs.json'))\n", "songs_df.columns = ['title']\n", "print('Total number of songs:', len(songs_df))\n", "songs_df.head()" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mean song title length: 22.43 chars\n" ] } ], "source": [ "song_len = (songs_df.title.str.len()).mean()\n", "print(f'mean song title length: {song_len:.2f} chars')" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "arxiv_titles:\n", " Deep Cuboid Detection: Beyond 2D Bounding Boxes\n", "Speed/accuracy trade-offs for modern convolutional o\n", "\n", "song_titles:\n", " Silver Lining\n", "Four Winds\n", "Half Love\n", "Sowa (Alex Gare\n" ] } ], "source": [ "arxiv_titles = '\\n'.join(arxiv_df.title)\n", "song_titles = '\\n'.join(songs_df.title)\n", "\n", "print('arxiv_titles:\\n', arxiv_titles[:100])\n", "print('\\nsong_titles:\\n', song_titles[:50])" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "arxiv_titles vocab_len: 111\n", "song_titles vocab_len: 142\n", "min vocab_len (and max batch_size): 111\n" ] } ], "source": [ "print('arxiv_titles vocab_len:', len(set(arxiv_titles)))\n", "print('song_titles vocab_len:', len(set(song_titles)))\n", "bs = min(len(set(arxiv_titles)), len(set(song_titles)))\n", "print('min vocab_len (and max batch_size):', bs)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": true }, "outputs": [], "source": [ "with open(os.path.join(DATA_DIR, 'arxiv_titles.txt'), 'w', encoding='utf-8') as f:\n", " f.write(arxiv_titles)\n", " \n", "with open(os.path.join(DATA_DIR, 'song_titles.txt'), 'w', encoding='utf-8') as f:\n", " f.write(song_titles)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Approach 1: Pretraining" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## DL paper titles -> song titles" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1074 }, "colab_type": "code", "id": "QWt__FLr-l18", "outputId": "26b91536-197e-4e2f-f773-1acc2870a428" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "10,002 texts collected.\n", "Training new model w/ 4-layer, 128-cell LSTMs\n", "Training on 562,837 character sequences.\n", "Epoch 1/10\n", "8794/8794 [==============================] - 406s 46ms/step - loss: 2.0727 - val_loss: 1.3352\n", "Epoch 2/10\n", "8794/8794 [==============================] - 398s 45ms/step - loss: 1.2272 - val_loss: 1.1812\n", "####################\n", "Temperature: 0.2\n", "####################\n", "Deep Learning of Computer Neural Networks with Deep Convolutional Neural Networks\n", "\n", "State Recognition of Control with Deep Learning of Convolutional Neural Networks\n", "\n", "State of Deep Learning of Convolutional Neural Networks for Structured Matrix Faction\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "Structural Sparse Shallow Analysis of Humanoratic Models for Prediction of Concerving Reconstruction and Typerspectral Regression\n", "\n", "A Deep Learning Learning of Image Sparse Recognition with Automatic Deep Learning\n", "\n", "Learning the Encoder Transformation in the Encoder in Stochastic Driven Selection of Consistent and Textures\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "A Droma of Treatforms: Can Learning to evide neural network for Two Imaging Structures\n", "\n", "Grutacurs Assolonithms with semantic Graphical RCV\n", "\n", "Deep Sentiment of Character ag Deval Improve Imaging intervention simulations\n", "\n", "Epoch 3/10\n", "8794/8794 [==============================] - 489s 56ms/step - loss: 1.1083 - val_loss: 1.1110\n", "Epoch 4/10\n", "8794/8794 [==============================] - 398s 45ms/step - loss: 1.0401 - val_loss: 1.0838\n", "####################\n", "Temperature: 0.2\n", "####################\n", "A Statistical Representation for Behavioral Subspace Classification\n", "\n", "Semantic Segmentation in Semantic Segmentation\n", "\n", "Semi-supervised Learning of Semantic Segmentation\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "Interpretation of Solvers from Saliency of Semantic Constrained Based on State Selection\n", "\n", "Statistical Layor Matching Algorithms Based on Propagation\n", "\n", "Network programming for the convolutional neural networks\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Focus trafform Generation in space from a Building in widding and syntactc, and mlo a\n", "\n", "InduV and Response Margin in Incremental Pindired Datasets for High Document Detection\n", "\n", "Mislocationzant Detection Using Deep-Grammatical Images\n", "\n", "Epoch 5/10\n", "8794/8794 [==============================] - 397s 45ms/step - loss: 0.9978 - val_loss: 1.0565\n", "Epoch 6/10\n", "8794/8794 [==============================] - 398s 45ms/step - loss: 0.9664 - val_loss: 1.0459\n", "####################\n", "Temperature: 0.2\n", "####################\n", "A New Semantic Parsing with Sparse Representations\n", "\n", "Product Networks for Predictive Adversarial Networks\n", "\n", "Learning to Predictive Adversarial Networks for Structured Sparsity Alignment and Transfer Learning\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "Synthesizing Random Forest Control for Context-Supervision Applications\n", "\n", "A New Head Network Function for Presence of Semantic Segmentation\n", "\n", "Coupled Data of Convolutional Neural Network Through Structured Detection in Supervision Make Recognition\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Deep Frame based Correction in Reasoning, Algorithms\n", "\n", "Madiods error closures\n", "\n", "Ehher-classifiers: Madgmability through Transfer Learning\n", "\n", "Epoch 7/10\n", "8794/8794 [==============================] - 398s 45ms/step - loss: 0.9214 - val_loss: 1.0258\n", "Epoch 8/10\n", "8794/8794 [==============================] - 398s 45ms/step - loss: 0.8805 - val_loss: 1.0247\n", "####################\n", "Temperature: 0.2\n", "####################\n", "A Deep Learning Approach to Deep Learning Models\n", "\n", "A New Method for Deep Learning Approaches for Deep Learning\n", "\n", "Semi-supervised learning for spectrum interaction and classification\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "A Deep Network Model for Spectrum Reconstruction and Selective Learning\n", "\n", "A Deep Learning Approach to Deep Neural Networks for Real-time Transfer Learning\n", "\n", "GPU-based Event Detection using Tree Simple Dependencies and Linear Density Estimation and Semi-supervised Learning of Medical Image Ret\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Event-Shallow Using Faster Curriculum Learning Lifte Predictability ildexming\n", "\n", "Semi-supervised dependent selection to drive modeling in the surget videos\n", "\n", "Long Short Memory Arlience of CNNs evolving Techniques\n", "\n", "Epoch 9/10\n", "8794/8794 [==============================] - 398s 45ms/step - loss: 0.8375 - val_loss: 1.0179\n", "Epoch 10/10\n", "8794/8794 [==============================] - 398s 45ms/step - loss: 0.7921 - val_loss: 1.0204\n", "####################\n", "Temperature: 0.2\n", "####################\n", "A Survey on Deep Neural Network for Sequence Labeling and Sequence Machine Translation\n", "\n", "A Comparison of Neural Network Architectures for Semantic Segmentation\n", "\n", "A Survey on Deep Learning Approach for Semantic Segmentation\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "A Multi-View Structure of Stream Selection in Continuous Functions and Minimization\n", "\n", "Semi-Supervised Learning for Learning a Multi-task Matching\n", "\n", "Automatic Structure of Semantic Parsing with Convolutional Neural Networks\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "IDAN: Play-Mo: Reading Hand synaphrous Conversations in Linear Regression with Optimal CT Dictionary Learning\n", "\n", "Penal-related to nissating harvetryctive multi-view system\n", "\n", "Scale-based Graph-Machine Learning Localization using Swarmprint Anatomety in First-Person Content\n", "\n" ] }, { "data": { "text/plain": [ "\"textgen.train_new_model(\\n arxiv_titles,\\n num_epochs=train_cfg['num_epochs'],\\n gen_epochs=train_cfg['gen_epochs'],\\n batch_size=train_cfg['batch_size'],\\n train_size=train_cfg['train_size'],\\n dropout=train_cfg['dropout'],\\n max_gen_length=train_cfg['max_gen_length'],\\n validation=train_cfg['validation'],\\n is_csv=train_cfg['is_csv'],\\n rnn_layers=model_cfg['rnn_layers'],\\n rnn_size=model_cfg['rnn_size'],\\n rnn_bidirectional=model_cfg['rnn_bidirectional'],\\n max_length=model_cfg['max_length'],\\n dim_embeddings=model_cfg['dim_embeddings'],\\n word_level=model_cfg['word_level']\\n)\"" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model_cfg = {\n", " 'rnn_size': 128,\n", " 'rnn_layers': 4,\n", " 'rnn_bidirectional': False, #True,\n", " 'max_length': int(paper_len),\n", " #'max_words': 10000,\n", " 'dim_embeddings': 100,\n", " 'word_level': False,\n", "}\n", "\n", "train_cfg = {\n", " 'num_epochs': 10,\n", " 'gen_epochs': 2,\n", " 'batch_size': bs, # 1024,\n", " 'train_size': 0.8,\n", " 'dropout': 0.0,\n", " 'max_gen_length': int(paper_len*2), #300,\n", " 'validation': True, # False,\n", " 'is_csv': False\n", "}\n", "\n", "paper_model_name = 'deep_paper_titles'\n", "textgen = textgenrnn(name=paper_model_name)\n", "\n", "textgen.train_from_file(\n", " file_path=os.path.join(DATA_DIR, 'arxiv_titles.txt'),\n", " new_model=True,\n", " num_epochs=train_cfg['num_epochs'],\n", " gen_epochs=train_cfg['gen_epochs'],\n", " batch_size=train_cfg['batch_size'],\n", " train_size=train_cfg['train_size'],\n", " dropout=train_cfg['dropout'],\n", " max_gen_length=train_cfg['max_gen_length'],\n", " validation=train_cfg['validation'],\n", " is_csv=train_cfg['is_csv'],\n", " rnn_layers=model_cfg['rnn_layers'],\n", " rnn_size=model_cfg['rnn_size'],\n", " rnn_bidirectional=model_cfg['rnn_bidirectional'],\n", " max_length=model_cfg['max_length'],\n", " dim_embeddings=model_cfg['dim_embeddings'],\n", " word_level=model_cfg['word_level']\n", ")" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "9,849 texts collected.\n", "Training on 184,005 character sequences.\n", "Epoch 1/1\n", "2875/2875 [==============================] - 131s 46ms/step - loss: 1.9834 - val_loss: 1.8403\n", "####################\n", "Temperature: 0.2\n", "####################\n", "Way (Roothine Remix)\n", "\n", "Will In The We (Kays Remix)\n", "\n", "Way To Feel (Bootleg)\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "Home (feat. K................5 ...........5.........................5...........5..............5................5....5..................\n", "\n", "Without Keep (Flum Remix)\n", "\n", "Inter Love\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Reson feat. Jeryni Map (Rockon Text Remix)\n", "\n", "Psners (GFURS Reqish)\n", "\n", "Lex The Wire In Get Oiv Like French\n", "\n" ] } ], "source": [ "textgen_song = textgenrnn(weights_path=os.path.join(DATA_DIR, 'models', f'{paper_model_name}_weights.hdf5'),\n", " vocab_path=os.path.join(DATA_DIR, 'models', f'{paper_model_name}_vocab.json'),\n", " config_path=os.path.join(DATA_DIR, 'models', f'{paper_model_name}_config.json'))\n", " \n", "textgen_song.train_from_file(\n", " file_path=os.path.join(DATA_DIR, 'song_titles.txt'),\n", " num_epochs=1,\n", " gen_epochs=1,\n", " batch_size=train_cfg['batch_size'],\n", " train_size=train_cfg['train_size'],\n", " dropout=0.5,\n", " max_gen_length=train_cfg['max_gen_length'],\n", " validation=train_cfg['validation'],\n", " is_csv=train_cfg['is_csv'],\n", " rnn_layers=model_cfg['rnn_layers'],\n", " rnn_size=model_cfg['rnn_size'],\n", " rnn_bidirectional=model_cfg['rnn_bidirectional'],\n", " max_length=model_cfg['max_length'],\n", " dim_embeddings=model_cfg['dim_embeddings'],\n", " word_level=model_cfg['word_level']\n", ")" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "####################\n", "Temperature: 0.2\n", "####################\n", "In The Wend (RAC Mix)\n", "\n", "Wond In The We (Roothing Remix)\n", "\n", "Stay (Robother Remix)\n", "\n", "Stay (Prod. by Know)\n", "\n", "Bellon The We Downt (feat. All In The Wook Remix)\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "Forest (Jaman Chainsmokers Remix)\n", "\n", "Way The Changer (Now Remix)\n", "\n", "When In Me (Kill Remix)\n", "\n", "Commer\n", "\n", "Green My Hand (Boul Club Remix)\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "About Un\n", "\n", "Glassica\n", "\n", "Ku Georgetting\n", "\n", "City (The Main Remax)\n", "\n", "Goodly Wwin Vice (Koelien Moh Remix)\n", "\n" ] } ], "source": [ "textgen_song.generate_samples(max_gen_length=100, n=5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Song titles -> Paper titles" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "9,849 texts collected.\n", "Training new model w/ 4-layer, 128-cell LSTMs\n", "Training on 184,530 character sequences.\n", "Epoch 1/10\n", "1662/1662 [==============================] - 37s 22ms/step - loss: 3.3772 - val_loss: 2.3540\n", "Epoch 2/10\n", "1662/1662 [==============================] - 36s 22ms/step - loss: 2.0871 - val_loss: 1.9189\n", "####################\n", "Temperature: 0.2\n", "####################\n", "Stronge (feat. Sean Brand)\n", "\n", "Searter (feat. Ander Brand)\n", "\n", "Wanter (feat. Aling Land)\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "Whe Somether\n", "\n", "No Love (Life Strac Remix)\n", "\n", "Comethan (Strive Remix)\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Wanny\n", "\n", "Night (Donk Coce Edit)\n", "\n", "The Pennega\n", "\n", "Epoch 3/10\n", "1662/1662 [==============================] - 36s 21ms/step - loss: 1.8194 - val_loss: 1.7822\n", "Epoch 4/10\n", "1662/1662 [==============================] - 37s 22ms/step - loss: 1.6628 - val_loss: 1.6967\n", "####################\n", "Temperature: 0.2\n", "####################\n", "Sunnast (Prod. by Big Shain)\n", "\n", "The Wild You (Solidis Sambi Remix)\n", "\n", "Better Way (feat. Sam Suntant)\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "Better With You (feat. Miesy)\n", "\n", "Say My Madion (Disco Bootleg)\n", "\n", "The Wild (feat. Kelex Boys)\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Nathing You Lover (Im Heud In My. Wanting \n", "\n", "Under Pratch (Benger Kelix Edit)\n", "\n", "Somk Live You Feat. K.E.A. & Motthra Have)\n", "\n", "Epoch 5/10\n", "1662/1662 [==============================] - 37s 22ms/step - loss: 1.5423 - val_loss: 1.6513\n", "Epoch 6/10\n", "1662/1662 [==============================] - 36s 22ms/step - loss: 1.4369 - val_loss: 1.6237\n", "####################\n", "Temperature: 0.2\n", "####################\n", "Stay (Sam Selektah Remix)\n", "\n", "The Starding (feat. Brasstree)\n", "\n", "Stay (feat. Lil Yachty) (Cassion Remix)\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "Feel In The Cold\n", "\n", "Come And Me (feat. Kele & Rae Mars)\n", "\n", "Drive Me Look (Lash Remix)\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Eugh feat. Blacks (produced by give Camman\n", "\n", "BelieveSpate\n", "\n", "Better Tois\n", "\n", "Epoch 7/10\n", "1662/1662 [==============================] - 37s 22ms/step - loss: 1.3404 - val_loss: 1.6078\n", "Epoch 8/10\n", "1662/1662 [==============================] - 36s 21ms/step - loss: 1.2503 - val_loss: 1.6263\n", "####################\n", "Temperature: 0.2\n", "####################\n", "Stay With Me\n", "\n", "Something Bout You (Remix)\n", "\n", "Sunshine\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "Hello (feat. Bright The Deep)\n", "\n", "Can't Hear Me There (feat. Arternan Ellie \n", "\n", "Hold On We're Going Home (feat. Antis Bloo\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Love Like Thus in Stills\n", "\n", "Moving Closer\n", "\n", "Cold Bob (Basswris Remix)\n", "\n", "Epoch 9/10\n", "1662/1662 [==============================] - 36s 21ms/step - loss: 1.1629 - val_loss: 1.6341\n", "Epoch 10/10\n", "1662/1662 [==============================] - 36s 21ms/step - loss: 1.0816 - val_loss: 1.6477\n", "####################\n", "Temperature: 0.2\n", "####################\n", "The One (Feat. Mathe Dayt)\n", "\n", "Superfriends (feat. Kendrick Lamar)\n", "\n", "Hold On We're Going Home (Dave Edwards Rem\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "The Startion feat. Zies Boy, Phonix Chorto\n", "\n", "Bend On My Mind (Marce Remix)\n", "\n", "Can't Help In Last (Like Sings Remix)\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Gone Probes feat. Embreesty (Laou Kniss & \n", "\n", "Black Bubble)\n", "\n", "Head Up (Loud Luxuse Remix)\n", "\n" ] } ], "source": [ "model_cfg = {\n", " 'rnn_size': 128,\n", " 'rnn_layers': 4,\n", " 'rnn_bidirectional': False, #True,\n", " 'max_length': int(song_len),\n", " #'max_words': 10000,\n", " 'dim_embeddings': 100,\n", " 'word_level': False,\n", "}\n", "\n", "train_cfg = {\n", " 'num_epochs': 10,\n", " 'gen_epochs': 2,\n", " 'batch_size': bs, # 1024,\n", " 'train_size': 0.8,\n", " 'dropout': 0.0,\n", " 'max_gen_length': int(song_len*2), #300,\n", " 'validation': True, # False,\n", " 'is_csv': False\n", "}\n", "\n", "song_model_name = 'deep_song_titles'\n", "textgen = textgenrnn(name=song_model_name)\n", "\n", "textgen.train_from_file(\n", " file_path=os.path.join(DATA_DIR, 'song_titles.txt'),\n", " new_model=True,\n", " num_epochs=train_cfg['num_epochs'],\n", " gen_epochs=train_cfg['gen_epochs'],\n", " batch_size=train_cfg['batch_size'],\n", " train_size=train_cfg['train_size'],\n", " dropout=train_cfg['dropout'],\n", " max_gen_length=train_cfg['max_gen_length'],\n", " validation=train_cfg['validation'],\n", " is_csv=train_cfg['is_csv'],\n", " rnn_layers=model_cfg['rnn_layers'],\n", " rnn_size=model_cfg['rnn_size'],\n", " rnn_bidirectional=model_cfg['rnn_bidirectional'],\n", " max_length=model_cfg['max_length'],\n", " dim_embeddings=model_cfg['dim_embeddings'],\n", " word_level=model_cfg['word_level']\n", ")" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "10,002 texts collected.\n", "Training on 562,523 character sequences.\n", "Epoch 1/1\n", "5067/5067 [==============================] - 106s 21ms/step - loss: 1.3301 - val_loss: 1.1896\n", "####################\n", "Temperature: 0.2\n", "####################\n", "A Simultaneous Search for Structure Learni\n", "\n", "A Neural Neural Networks for Multi-label D\n", "\n", "A Statistical Search for Structure Models \n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "A Neural Networks for Real-time Cluster fo\n", "\n", "Bein Low-Reduced Prediction in Multi-modal\n", "\n", "Domain Function Using Deep Learning with B\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Multi-Agents independence propagator seep \n", "\n", "Typeoring-Level Mundems\n", "\n", "Intervesconronquent Variatorits for RalisN\n", "\n" ] } ], "source": [ "textgen_paper = textgenrnn(weights_path=os.path.join(DATA_DIR, 'models', f'{song_model_name}_weights.hdf5'),\n", " vocab_path=os.path.join(DATA_DIR, 'models', f'{song_model_name}_vocab.json'),\n", " config_path=os.path.join(DATA_DIR, 'models', f'{song_model_name}_config.json'))\n", " \n", "textgen_paper.train_from_file(\n", " file_path=os.path.join(DATA_DIR, 'arxiv_titles.txt'),\n", " num_epochs=1,\n", " gen_epochs=1,\n", " batch_size=train_cfg['batch_size'],\n", " train_size=train_cfg['train_size'],\n", " dropout=0.9,\n", " max_gen_length=train_cfg['max_gen_length'],\n", " validation=train_cfg['validation'],\n", " is_csv=train_cfg['is_csv'],\n", " rnn_layers=model_cfg['rnn_layers'],\n", " rnn_size=model_cfg['rnn_size'],\n", " rnn_bidirectional=model_cfg['rnn_bidirectional'],\n", " max_length=model_cfg['max_length'],\n", " dim_embeddings=model_cfg['dim_embeddings'],\n", " word_level=model_cfg['word_level']\n", ")" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "####################\n", "Temperature: 0.5\n", "####################\n", "A Linear Learning Framework for Person Re-identification of Stochastic Sensors Using Model Predict\n", "\n", "A Normalized Constrained End-to-End Linear Gradient Face Regression with Deep Learning and Sensor \n", "\n", "A Discovery of Finiter Grade for Learning for Resolution Models\n", "\n", "A Deep Neural Networks for Fully Context Placking\n", "\n", "Multi-labeling Large Sensing for Multi-view Constrained Residual Learning for Betweed Programs of \n", "\n", "####################\n", "Temperature: 0.7\n", "####################\n", "Neural Networks and Function Processing\n", "\n", "Exploining Deep Learning for Face extraction of Structure Detection\n", "\n", "Dlatchion Detection for Learning Data for Unsupervised Learning in Visual Recognizing\n", "\n", "A tro Constrained Neighbor Models\n", "\n", "Trick Prediction with Minimum Band Features Model for Neural Language Modeling and Hate Tracking D\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Multimor: The CLAM-IO Ranch Scoculus Tracking\n", "\n", "Motion Large Veor Multillal Recognization Framework for Recognizing Large-scale Information for Fa\n", "\n", "Neural Factorizits Prices\n", "\n", "Groundded Robust Waras\n", "\n", "Non-Seats and Pening Based Netwark Reduced Low-Rank Tabdling\n", "\n" ] } ], "source": [ "textgen_paper.generate_samples(max_gen_length=100, temperatures=[0.5, 0.7, 1.0], n=5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Approach 2: Mixed" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "collapsed": true }, "outputs": [], "source": [ "with open(os.path.join(DATA_DIR, 'mixed_titles.txt'), 'w', encoding='utf-8') as f:\n", " f.write(arxiv_titles + '\\n' + song_titles)" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "19,852 texts collected.\n", "Training new model w/ 4-layer, 128-cell LSTMs\n", "Training on 747,719 character sequences.\n", "Epoch 1/2\n", "11683/11683 [==============================] - 147s 13ms/step - loss: 1.7612 - val_loss: 1.5157\n", "####################\n", "Temperature: 0.2\n", "####################\n", "Based Recognition in Deep Neural Network for Spe\n", "\n", "A State Convolutional Network for State Convolut\n", "\n", "Deep Neural Networks\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "A Shode Linear Sequential Streaming Remix)\n", "\n", "An Every of Localization Me Down (feat. A. Gradi\n", "\n", "An Estimation of a Grammodition of Evolutional N\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "C-Neclarided Localized Dispricon Layeroa)\n", "\n", "Arn't Frame Continuous intelliative for Video Re\n", "\n", "Intacle Aggrix (Low Freed in Work\n", "\n", "Epoch 2/2\n", "11683/11683 [==============================] - 143s 12ms/step - loss: 1.4492 - val_loss: 1.3971\n", "####################\n", "Temperature: 0.2\n", "####################\n", "Multi-Task Learning for Sparse Remix)\n", "\n", "Exploring with Recognition in Context for Neural\n", "\n", "Recognition for Multi-modal Recognition for Spar\n", "\n", "####################\n", "Temperature: 0.5\n", "####################\n", "Improved Translations\n", "\n", "Structural Networks\n", "\n", "Multimodal Segmentation of End-to-End Cardand (T\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Whard and Me\n", "\n", "Line Cell (Prediction Networks for Supervised-Pe\n", "\n", "UltTrims for Visual Loses Remix)\n", "\n" ] } ], "source": [ "model_cfg = {\n", " 'rnn_size': 128,\n", " 'rnn_layers': 4,\n", " 'rnn_bidirectional': False, #True,\n", " 'max_length': 5, #40,\n", " 'max_words': 10000,\n", " 'dim_embeddings': 100,\n", " 'word_level': False,\n", "}\n", "\n", "train_cfg = {\n", " 'num_epochs': 2, # 10,\n", " 'gen_epochs': 1,\n", " 'batch_size': 64, # 1024,\n", " 'train_size': 0.8,\n", " 'dropout': 0.5, # 0.0,\n", " 'max_gen_length': 50, #300,\n", " 'validation': True, # False,\n", " 'is_csv': False\n", "}\n", "\n", "mixed_model_name = 'mixed'\n", "textgen = textgenrnn(name=mixed_model_name)\n", "textgen.train_from_file(\n", " file_path=os.path.join(DATA_DIR, 'mixed_titles.txt'),\n", " new_model=True,\n", " num_epochs=train_cfg['num_epochs'],\n", " gen_epochs=train_cfg['gen_epochs'],\n", " batch_size=train_cfg['batch_size'],\n", " train_size=train_cfg['train_size'],\n", " dropout=train_cfg['dropout'],\n", " max_gen_length=train_cfg['max_gen_length'],\n", " validation=train_cfg['validation'],\n", " is_csv=train_cfg['is_csv'],\n", " rnn_layers=model_cfg['rnn_layers'],\n", " rnn_size=model_cfg['rnn_size'],\n", " rnn_bidirectional=model_cfg['rnn_bidirectional'],\n", " max_length=model_cfg['max_length'],\n", " dim_embeddings=model_cfg['dim_embeddings'],\n", " word_level=model_cfg['word_level']\n", ")" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "####################\n", "Temperature: 0.5\n", "####################\n", "Experiment Learning Embedding for Prediction for Prediction with Media Segmentation\n", "\n", "Metric Interpretable Generation\n", "\n", "DeepMe (feat. Remix)\n", "\n", "On Me (feat. Andreak (feat. Change Weight (Feat. The Minimal Recognition for Deep Neural networks \n", "\n", "Good (Feat. The Love (feat. Bundit Detection\n", "\n", "####################\n", "Temperature: 0.7\n", "####################\n", "Framework for Subspace (Dance of Resolutional Networks\n", "\n", "Shot Caption\n", "\n", "Bie Detection with Generation On (Feat Nonless Free Box Story Method for Parallel recurrent Neural\n", "\n", "Everything\n", "\n", "Survey\n", "\n", "####################\n", "Temperature: 1.0\n", "####################\n", "Gradients\n", "\n", "Lesion Restarding with a Wild\n", "\n", "Recognitional similarity\n", "\n", "Identification Networks\n", "\n", "Face Recognizing Label 1017Id2 ME Romix Remix)\n", "\n" ] } ], "source": [ "textgen.generate_samples(max_gen_length=100, n=5, temperatures=[0.5, 0.7, 1.0])" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# Best Of\n", "\n", "Default textgenrnn settings (2 epochs):\n", "\n", "- Flexing To The Study [temp 0.5]\n", "- Embedding feat. Ceyta Readm [temp 1.0]\n", "\n", "Rebalancing songs and papers + max_length of 5 + not bi-directional + dropout of 0.5 + short max_gen_length (2 epochs):\n", "\n", "- Strong (feat. Salial Networks) [temp 0.2]\n", "- Moon (feat. Ligoning for Structured Remix) [temp 0.2]\n", "- Completion and Continuous remix [temp 0.5]\n", "\n", "Same as last, but max_length of 10 + dropout of 0.1:\n", "\n", "- State (Feat. Kanna Alignment Algorithms) [temp 0.2]\n", "- Like You Way To (Original Learning from a Hadcer convex framework) [temp 0.9]\n", " - reminds me of \"(original mix)\" or \"(original score)\" tracks\n", "\n", "Same as last, but max_length of 5:\n", "\n", "- Space Camera & Anomaly [temp 1.0]\n", " - could be a band name\n", "- Correction in Remix [temp 0.5]\n", "- Automatic Samples for sea Remix [temp 0.8]\n", "- Silver (Sving Recognitive Network prooft Remix) [temp 0.8]\n", " - the easiest way to make something a song is to add 'remix' to the end of it\n", "- A Subspaces [temp 0.5]\n", "\n", "Lost track of these:\n", "\n", "- Learning Theory (Live Self Model) [temp 0.2]\n", "- A Problems of Gents [temp 1.0]\n", "- Task2Quous Dreams [temp 1.0]\n", "- Drop Loud [temp 1.0]\n", "- DeepSEGK: A Reconstruction [temp 1.0]\n", " - could be an album title\n", " \n", "Same as 'space camera' one, but dropout of 0.5:\n", "\n", "- DeepMe (feat. Remix) [temp 0.5]\n", "- Framework for Subspace (Dance of Resolutional Networks) [temp 0.7]\n", "- An Every of Localization Me Down (feat. A. Gradi) [temp 0.5]\n", "\n", "deep_paper_titles only:\n", "(not mixed with song titles at all... but they still seem musical)\n", "\n", "- Deep Sentiment of Character [temp 1.0]\n", "- Semantic Segmentation in Semantic Segmentation [temp 0.2]\n", " \n", " \n", "## Takeaways\n", "- More epochs didn't help\n", "- Balancing number of titles from each set helped\n", "- Shane mentioned that she (accidentally) pretrained on metal bands and then used transfer learning to add ice cream flavors - however, I couldn't get this approach to work better than just mixing the two sets together and training in one go" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "textgenrnn_deep_song_titles", "provenance": [], "version": "0.3.2" }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.2" } }, "nbformat": 4, "nbformat_minor": 1 }