{"id":438,"date":"2022-06-11T21:46:47","date_gmt":"2022-06-11T13:46:47","guid":{"rendered":"https:\/\/scutvk.cn\/?p=438"},"modified":"2022-06-13T16:13:19","modified_gmt":"2022-06-13T08:13:19","slug":"pytorch-%e5%88%92%e5%88%86%e6%95%b0%e6%8d%ae%e9%9b%86torch-utils-data-dataset-dataset","status":"publish","type":"post","link":"https:\/\/scutvk.cn\/?p=438","title":{"rendered":"Pytorch \u5212\u5206\u6570\u636e\u96c6torch.utils.data.dataset.Dataset"},"content":{"rendered":"\n<p>\u5173\u952e\u5b57\uff1a\u5212\u5206\u6570\u636e\u96c6\u3001torch.utils.data.dataset.Dataset\u3001torch.utils.data.random_split<\/p>\n\n\n\n<p>\u5b9e\u73b0\u6548\u679c\uff1a\u5c06Dataset\u7c7b\u6309\u6307\u5b9a\u6bd4\u4f8b\u5212\u5206\u4e3a\u8bad\u7ec3\u96c6\u3001\u9a8c\u8bc1\u96c6\uff08\u548c\u6d4b\u8bd5\u96c6\uff09<\/p>\n\n\n\n<p>\u4e3a\u5b9e\u73b0\u6b64\u76ee\u7684\uff0c\u6211\u4eec\u4f7f\u7528\u5230pytorch\u7684\u51fd\u6570torch.utils.data.random_split()<\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<p class=\"has-large-font-size\"><strong>\u51fd\u6570\u5c1d\u8bd5\uff1a<\/strong><\/p>\n\n\n\n<p><strong>random_split(dataset, lengths, generator)<\/strong><\/p>\n\n\n\n<p><strong>\u8f93\u5165\u53c2\u6570\uff1a<\/strong><\/p>\n\n\n\n<p>dataset\uff0c\u7c7b\u578b\uff1atorch.utils.data.dataset.Dataset\uff0c\u6b32\u5212\u5206\u7684\u6570\u636e\u96c6<\/p>\n\n\n\n<p>lengths\uff0c\u7c7b\u578b\uff1asequence\uff0c\u4f8b\u5982[7, 3]\uff0c\u6ce8\u610f\u610f\u601d<strong>\u4e0d\u662f<\/strong>\u628a\u6570\u636e\u96c6\u63097\uff1a3\u6bd4\u4f8b\u5212\u5206\uff0c\u800c\u662f\u5212\u5206\u4e3a<strong>\u5927\u5c0f\u5206\u522b<\/strong>\u4e3a7\u548c3\u7684\u6570\u636e\u96c6\u3002<\/p>\n\n\n\n<p>generator\uff0c\u7c7b\u578b\uff1atorch.Generator\uff0c\u968f\u673a\u6570\u751f\u6210\u5668\uff0c<strong>\u5e26\u9ed8\u8ba4\u53c2\u6570<\/strong>generator = default_generator<\/p>\n\n\n\n<p><strong>\u8fd4\u56de\u53c2\u6570\uff1a<\/strong><\/p>\n\n\n\n<p>torch.utils.data.dataset.Subset\u7684\u6570\u7ec4<\/p>\n\n\n\n<p>\u5b98\u65b9\u4f8b\u5b50\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>random_split(range(10), &#91;3, 7], generator=torch.Generator().manual_seed(42))<\/code><\/pre>\n\n\n\n<figure class=\"wp-block-image size-full is-resized\"><img decoding=\"async\" loading=\"lazy\" src=\"https:\/\/x2.mday.top\/wp-content\/uploads\/2022\/06\/image-3.png\" alt=\"\" class=\"wp-image-440\" width=\"704\" height=\"143\" srcset=\"https:\/\/scutvk.cn\/wp-content\/uploads\/2022\/06\/image-3.png 938w, https:\/\/scutvk.cn\/wp-content\/uploads\/2022\/06\/image-3-300x61.png 300w, https:\/\/scutvk.cn\/wp-content\/uploads\/2022\/06\/image-3-768x156.png 768w, https:\/\/scutvk.cn\/wp-content\/uploads\/2022\/06\/image-3-769x157.png 769w\" sizes=\"(max-width: 704px) 100vw, 704px\" \/><\/figure>\n\n\n\n<p>\u8fd4\u56de\u7684\u662f2\u4e2adataset.Subset<\/p>\n\n\n\n<p>\u7b14\u8005\u4f8b\u5b50\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code># load dataset\ntotal_dataset = mrds.TxtDataset(neg_loc=\"..\/raw_data\/rt-polaritydata\/rt-polaritydata\/rt-polarity.neg\", pos_loc=\"..\/raw_data\/rt-polaritydata\/rt-polaritydata\/rt-polarity.pos\", word2index=glove_vocab_dic) # \u8fd4\u56de\u4e86\u4e00\u4e2aDataset\ntrain_length = int(total_dataset.__len__() * 0.7) #  \u83b7\u53d6\u8bad\u7ec3\u96c6\u957f\u5ea6\nval_length = total_dataset.__len__() - train_length #  \u83b7\u53d6\u9a8c\u8bc1\u96c6\u957f\u5ea6\ntrain_dataset, val_dataset = random_split(dataset=total_dataset, lengths=&#91;train_length, val_length])<\/code><\/pre>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" loading=\"lazy\" width=\"602\" height=\"101\" src=\"https:\/\/x2.mday.top\/wp-content\/uploads\/2022\/06\/image-4.png\" alt=\"\" class=\"wp-image-444\" srcset=\"https:\/\/scutvk.cn\/wp-content\/uploads\/2022\/06\/image-4.png 602w, https:\/\/scutvk.cn\/wp-content\/uploads\/2022\/06\/image-4-300x50.png 300w\" sizes=\"(max-width: 602px) 100vw, 602px\" \/><\/figure>\n\n\n\n<p>\u89c2\u5bdftrain_dataset\u3001val_dataset\u4e24\u4e2aSubset\u7684\u6210\u5458\uff0c\u53d1\u73b0\u5185\u7531dataset\u6210\u5458\uff0c\u8fd9\u4fbf\u662f\u6211\u4eec\u8981\u7684\u3001\u5206\u5272\u540e\u7684\u8bad\u7ec3\u96c6\u3002<\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" loading=\"lazy\" width=\"816\" height=\"144\" src=\"https:\/\/x2.mday.top\/wp-content\/uploads\/2022\/06\/image-5.png\" alt=\"\" class=\"wp-image-448\" srcset=\"https:\/\/scutvk.cn\/wp-content\/uploads\/2022\/06\/image-5.png 816w, https:\/\/scutvk.cn\/wp-content\/uploads\/2022\/06\/image-5-300x53.png 300w, https:\/\/scutvk.cn\/wp-content\/uploads\/2022\/06\/image-5-768x136.png 768w, https:\/\/scutvk.cn\/wp-content\/uploads\/2022\/06\/image-5-769x136.png 769w\" sizes=\"(max-width: 816px) 100vw, 816px\" \/><\/figure>\n\n\n\n<p>\u52a0\u4e0a\u4e24\u7aef\u6e90\u7801<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>train_dataset = train_dataset.dataset\r\nval_dataset = val_dataset.dataset<\/code><\/pre>\n\n\n\n<p>\u5927\u529f\u544a\u6210\u3002<\/p>\n\n\n\n<hr class=\"wp-block-separator has-alpha-channel-opacity\"\/>\n\n\n\n<p class=\"has-large-font-size\"><strong>\u603b\u7ed3\uff1a<\/strong><\/p>\n\n\n\n<p>\u76f4\u63a5\u642c\u7816\u53ef\u7528\u3001\u603b\u6e90\u7801\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code># load dataset\ntotal_dataset = mrds.TxtDataset(neg_loc=\"..\/raw_data\/rt-polaritydata\/rt-polaritydata\/rt-polarity.neg\", pos_loc=\"..\/raw_data\/rt-polaritydata\/rt-polaritydata\/rt-polarity.pos\", word2index=glove_vocab_dic) # \u8fd4\u56de\u4e86\u4e00\u4e2aDataset\ntrain_length = int(total_dataset.__len__() * 0.7) #  \u83b7\u53d6\u8bad\u7ec3\u96c6\u957f\u5ea6\nval_length = total_dataset.__len__() - train_length #  \u83b7\u53d6\u9a8c\u8bc1\u96c6\u957f\u5ea6\ntrain_dataset, val_dataset = random_split(dataset=total_dataset, lengths=&#91;train_length, val_length])\ntrain_dataset = train_dataset.dataset\nval_dataset = val_dataset.dataset<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u5173\u952e\u5b57\uff1a\u5212\u5206\u6570\u636e\u96c6\u3001torch.utils.dat&hellip;<\/p>\n","protected":false},"author":1,"featured_media":440,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"jetpack_publicize_message":"","jetpack_is_tweetstorm":false,"jetpack_publicize_feature_enabled":true,"jetpack_social_post_already_shared":false,"jetpack_social_options":{"image_generator_settings":{"template":"highway","enabled":false}}},"categories":[10,12,2],"tags":[],"jetpack_publicize_connections":[],"jetpack_featured_media_url":"https:\/\/scutvk.cn\/wp-content\/uploads\/2022\/06\/image-3.png","_links":{"self":[{"href":"https:\/\/scutvk.cn\/index.php?rest_route=\/wp\/v2\/posts\/438"}],"collection":[{"href":"https:\/\/scutvk.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/scutvk.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/scutvk.cn\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/scutvk.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=438"}],"version-history":[{"count":7,"href":"https:\/\/scutvk.cn\/index.php?rest_route=\/wp\/v2\/posts\/438\/revisions"}],"predecessor-version":[{"id":449,"href":"https:\/\/scutvk.cn\/index.php?rest_route=\/wp\/v2\/posts\/438\/revisions\/449"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/scutvk.cn\/index.php?rest_route=\/wp\/v2\/media\/440"}],"wp:attachment":[{"href":"https:\/\/scutvk.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=438"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/scutvk.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=438"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/scutvk.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=438"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}