{"id":3337,"date":"2022-03-31T16:38:13","date_gmt":"2022-03-31T08:38:13","guid":{"rendered":"http:\/\/139.9.1.231\/?p=3337"},"modified":"2022-03-31T16:38:15","modified_gmt":"2022-03-31T08:38:15","slug":"torch-optim-lr_schedule","status":"publish","type":"post","link":"http:\/\/139.9.1.231\/index.php\/2022\/03\/31\/torch-optim-lr_schedule\/","title":{"rendered":"torch.optim.lr_scheduler \u5b66\u4e60\u7387\u7684\u52a8\u6001\u8c03\u6574"},"content":{"rendered":"\n

\u5f53\u6211\u4eec\u5728\u8bad\u7ec3\u795e\u7ecf\u7f51\u7edc\u65f6\u5019\uff0c\u5b66\u4e60\u7387\u4f5c\u4e3a\u4e00\u4e2a\u8d85\u53c2\u6570\uff0c\u9700\u8981\u6211\u4eec\u6307\u5b9a\uff0c\u5982\u679clr\u8fc7\u5927\uff0c\u4f1a\u5bfc\u81f4\u6a21\u578b\u4e0d\u6536\u655b\uff0c\u9707\u8361\uff0c\u800c\u5982\u679clr\u5f88\u5c0f\uff0c\u4f1a\u5bfc\u81f4\u6536\u655b\u6162\uff0c\u8bad\u7ec3\u65f6\u95f4\u957f\u3002<\/p>\n\n\n\n

\u4e00\u4e2a\u6700\u666e\u904d\u7684\u60f3\u6cd5\uff0c\u521d\u59cb\u65f6lr\u8bbe\u7f6e\u5927\u4e00\u4e9b\uff0c\u968f\u7740epoch\u7684\u589e\u52a0\uff0clr\u9010\u6e10\u4e0b\u964d\u3002<\/p>\n\n\n\n

torch.optim.lr_scheduler<\/code>\u6a21\u5757\u63d0\u4f9b\u4e86\u4e00\u4e9b\u6839\u636eepoch\u8bad\u7ec3\u6b21\u6570\u6765\u8c03\u6574\u5b66\u4e60\u7387\uff08learning rate\uff09\u7684\u65b9\u6cd5<\/p>\n\n\n\n

\u6e90\u7801\uff1ahttps:\/\/pytorch.org\/docs\/stable\/_modules\/torch\/optim\/lr_scheduler.html<\/a><\/p>\n\n\n\n

torch<\/a>.optim.lr_scheduler\u4e2d\u5927\u90e8\u5206\u8c03\u6574\u5b66\u4e60\u7387\u7684\u65b9\u6cd5\u90fd\u662f\u6839\u636eepoch\u8bad\u7ec3\u6b21\u6570\uff0c\u8fd9\u91cc\u4ecb\u7ecd\u5e38\u89c1\u7684\u51e0\u79cd\u65b9\u6cd5\uff0c\u5176\u4ed6\u65b9\u6cd5\u4ee5\u540e\u7528\u5230\u518d\u8865\u5145\u3002
\u8981\u4e86\u89e3\u6bcf\u4e2a\u7c7b\u7684\u66f4\u65b0\u7b56\u7565<\/strong>\uff0c\u53ef\u76f4\u63a5\u67e5\u770b\u5b98\u7f51doc\u4e2d\u7684\u6e90\u7801<\/a>\uff0c\u6bcf\u7c7b\u90fd\u6709\u4e2aget_lr<\/code>\u65b9\u6cd5\uff0c\u5b9a\u4e49\u4e86\u66f4\u65b0\u7b56\u7565<\/p>\n\n\n\n

1\u3001torch.optim.lr_scheduler.LambdaLR<\/p>\n\n\n\n

  >>> # Assuming optimizer has two groups.\r\n        >>> lambda1 = lambda epoch: epoch \/\/ 30\r\n        >>> lambda2 = lambda epoch: 0.95 ** epoch\r\n        >>> scheduler = LambdaLR(optimizer, lr_lambda=[lambda1, lambda2])\r\n        >>> for epoch in range(100):\r\n        >>>     train(...)\r\n        >>>     validate(...)\r\n        >>>     scheduler.step()<\/code><\/pre>\n\n\n\n
class torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda, last_epoch=-1)<\/p>\n\n\n\n
\u53c2\u6570\uff1a<\/strong><\/p>\n\n\n\n
  1. optimizer \uff08Optimizer\uff09\uff1a\u8981\u66f4\u6539\u5b66\u4e60\u7387\u7684\u4f18\u5316\u5668\uff1b<\/li>
  2. lr_lambda\uff08function or list\uff09\uff1a\u6839\u636eepoch\u8ba1\u7b97\u03bb<\/em>\u7684\u51fd\u6570\uff1b\u6216\u8005\u662f\u4e00\u4e2alist<\/code>\u7684\u8fd9\u6837\u7684function\uff0c\u5206\u522b\u8ba1\u7b97\u5404\u4e2aparameter groups\u7684\u5b66\u4e60\u7387\u66f4\u65b0\u7528\u5230\u7684\u03bb<\/em>\uff1b<\/li>
  3. last_epoch \uff08int\uff09\uff1a\u6700\u540e\u4e00\u4e2aepoch\u7684index\uff0c\u5982\u679c\u662f\u8bad\u7ec3\u4e86\u5f88\u591a\u4e2aepoch\u540e\u4e2d\u65ad\u4e86\uff0c\u7ee7\u7eed\u8bad\u7ec3\uff0c\u8fd9\u4e2a\u503c\u5c31\u7b49\u4e8e\u52a0\u8f7d\u7684\u6a21\u578b\u7684epoch\u3002\u9ed8\u8ba4\u4e3a-1\u8868\u793a\u4ece\u5934\u5f00\u59cb\u8bad\u7ec3\uff0c\u5373\u4eceepoch=1\u5f00\u59cb\u3002<\/li><\/ol>\n\n\n\n
    \u66f4\u65b0\u7b56\u7565\uff1a<\/strong>
                         new _ l r = \u03bb<\/em>\u00d7initial<\/em>_lr<\/em>
    \u5176\u4e2dnew<\/em>_lr<\/em>\u662f\u5f97\u5230\u7684\u65b0\u7684\u5b66\u4e60\u7387\uff0cinitial<\/em>_lr<\/em>\u662f\u521d\u59cb\u7684\u5b66\u4e60\u7387\uff0c\u03bb\u662f\u901a\u8fc7\u53c2\u6570lr_lambda\u548cepoch\u5f97\u5230\u7684\u3002<\/p>\n\n\n\n
    import torch\r\nimport torch.nn as nn\r\nfrom torch.optim.lr_scheduler import LambdaLR\r\n\r\ninitial_lr = 0.1\r\n\r\nclass model(nn.Module):\r\n    def __init__(self):\r\n        super().__init__()\r\n        self.conv1 = nn.Conv2d(in_channels=3, out_channels=3, kernel_size=3)\r\n\r\n    def forward(self, x):\r\n        pass\r\n\r\nnet_1 = model()\r\n\r\noptimizer_1 = torch.optim.Adam(net_1.parameters(), lr = initial_lr)\r\nscheduler_1 = LambdaLR(optimizer_1, lr_lambda=lambda epoch: 1\/(epoch+1))\r\n\r\nprint(\"\u521d\u59cb\u5316\u7684\u5b66\u4e60\u7387\uff1a\", optimizer_1.defaults['lr'])\r\n\r\nfor epoch in range(1, 11):\r\n    # train\r\n\r\n    optimizer_1.zero_grad()\r\n    optimizer_1.step()\r\n    print(\"\u7b2c%d\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a%f\" % (epoch, optimizer_1.param_groups[0]['lr']))\r\n    scheduler_1.step()<\/code><\/pre>\n\n\n\n
    \u521d\u59cb\u5316\u7684\u5b66\u4e60\u7387\uff1a 0.1\r\n\u7b2c1\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.100000\r\n\u7b2c2\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.050000\r\n\u7b2c3\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.033333\r\n\u7b2c4\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.025000\r\n\u7b2c5\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.020000\r\n\u7b2c6\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.016667\r\n\u7b2c7\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.014286\r\n\u7b2c8\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.012500\r\n\u7b2c9\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.011111\r\n\u7b2c10\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.010000<\/code><\/pre>\n\n\n\n
    class torch.optim.lr_scheduler.StepLR(optimizer, step_size, gamma=0.1, last_epoch=-1)<\/p>\n\n\n\n
    \"\"<\/figure>\n\n\n\n
    \u53c2\u6570\uff1a<\/strong><\/p>\n\n\n\n
    1. optimizer \uff08Optimizer\uff09\uff1a\u8981\u66f4\u6539\u5b66\u4e60\u7387\u7684\u4f18\u5316\u5668\uff1b<\/li>
    2. step_size\uff08int\uff09\uff1a\u6bcf\u8bad\u7ec3step_size\u4e2aepoch\uff0c\u66f4\u65b0\u4e00\u6b21\u53c2\u6570\uff1b<\/li>
    3. gamma\uff08float\uff09\uff1a\u66f4\u65b0lr\u7684\u4e58\u6cd5\u56e0\u5b50\uff1b<\/li>
    4. last_epoch \uff08int\uff09\uff1a\u6700\u540e\u4e00\u4e2aepoch\u7684index\uff0c\u5982\u679c\u662f\u8bad\u7ec3\u4e86\u5f88\u591a\u4e2aepoch\u540e\u4e2d\u65ad\u4e86\uff0c\u7ee7\u7eed\u8bad\u7ec3\uff0c\u8fd9\u4e2a\u503c\u5c31\u7b49\u4e8e\u52a0\u8f7d\u7684\u6a21\u578b\u7684epoch\u3002\u9ed8\u8ba4\u4e3a-1\u8868\u793a\u4ece\u5934\u5f00\u59cb\u8bad\u7ec3\uff0c\u5373\u4eceepoch=1\u5f00\u59cb\u3002<\/li><\/ol>\n\n\n\n
      import torch\r\nimport torch.nn as nn\r\nfrom torch.optim.lr_scheduler import StepLR\r\nimport itertools\r\n\r\n\r\ninitial_lr = 0.1\r\n\r\nclass model(nn.Module):\r\n    def __init__(self):\r\n        super().__init__()\r\n        self.conv1 = nn.Conv2d(in_channels=3, out_channels=3, kernel_size=3)\r\n\r\n    def forward(self, x):\r\n        pass\r\n\r\nnet_1 = model()\r\n\r\noptimizer_1 = torch.optim.Adam(net_1.parameters(), lr = initial_lr)\r\nscheduler_1 = StepLR(optimizer_1, step_size=3, gamma=0.1)\r\n\r\nprint(\"\u521d\u59cb\u5316\u7684\u5b66\u4e60\u7387\uff1a\", optimizer_1.defaults['lr'])\r\n\r\nfor epoch in range(1, 11):\r\n    # train\r\n\r\n    optimizer_1.zero_grad()\r\n    optimizer_1.step()\r\n    print(\"\u7b2c%d\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a%f\" % (epoch, optimizer_1.param_groups[0]['lr']))\r\n    scheduler_1.step()<\/code><\/pre>\n\n\n\n
      \u521d\u59cb\u5316\u7684\u5b66\u4e60\u7387\uff1a 0.1\r\n\u7b2c1\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.100000\r\n\u7b2c2\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.100000\r\n\u7b2c3\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.100000\r\n\u7b2c4\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.010000\r\n\u7b2c5\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.010000\r\n\u7b2c6\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.010000\r\n\u7b2c7\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.001000\r\n\u7b2c8\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.001000\r\n\u7b2c9\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.001000\r\n\u7b2c10\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.000100<\/code><\/pre>\n\n\n\n
      class torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones, gamma=0.1, last_epoch=-1)<\/p>\n\n\n\n
      \u6bcf\u6b21\u9047\u5230milestones<\/code>\u4e2d\u7684epoch\uff0c\u505a\u4e00\u6b21\u66f4\u65b0\uff1a<\/p>\n\n\n\n
      \"\"<\/figure>\n\n\n\n
      \u53c2\u6570\uff1a<\/strong><\/p>\n\n\n\n
      1. optimizer \uff08Optimizer\uff09\uff1a\u8981\u66f4\u6539\u5b66\u4e60\u7387\u7684\u4f18\u5316\u5668\uff1b<\/li>
      2. milestones\uff08list\uff09\uff1a\u9012\u589e\u7684list\uff0c\u5b58\u653e\u8981\u66f4\u65b0lr\u7684epoch\uff1b<\/li>
      3. gamma\uff08float\uff09\uff1a\u66f4\u65b0lr\u7684\u4e58\u6cd5\u56e0\u5b50\uff1b<\/li>
      4. last_epoch \uff08int\uff09\uff1a\u6700\u540e\u4e00\u4e2aepoch\u7684index\uff0c\u5982\u679c\u662f\u8bad\u7ec3\u4e86\u5f88\u591a\u4e2aepoch\u540e\u4e2d\u65ad\u4e86\uff0c\u7ee7\u7eed\u8bad\u7ec3\uff0c\u8fd9\u4e2a\u503c\u5c31\u7b49\u4e8e\u52a0\u8f7d\u7684\u6a21\u578b\u7684epoch\u3002\u9ed8\u8ba4\u4e3a-1\u8868\u793a\u4ece\u5934\u5f00\u59cb\u8bad\u7ec3\uff0c\u5373\u4eceepoch=1\u5f00\u59cb\u3002<\/li><\/ol>\n\n\n\n
        import torch\r\nimport torch.nn as nn\r\nfrom torch.optim.lr_scheduler import MultiStepLR\r\nimport itertools\r\n\r\n\r\ninitial_lr = 0.1\r\n\r\nclass model(nn.Module):\r\n    def __init__(self):\r\n        super().__init__()\r\n        self.conv1 = nn.Conv2d(in_channels=3, out_channels=3, kernel_size=3)\r\n\r\n    def forward(self, x):\r\n        pass\r\n\r\nnet_1 = model()\r\n\r\noptimizer_1 = torch.optim.Adam(net_1.parameters(), lr = initial_lr)\r\nscheduler_1 = MultiStepLR(optimizer_1, milestones=[3, 7], gamma=0.1)\r\n\r\nprint(\"\u521d\u59cb\u5316\u7684\u5b66\u4e60\u7387\uff1a\", optimizer_1.defaults['lr'])\r\n\r\nfor epoch in range(1, 11):\r\n    # train\r\n\r\n    optimizer_1.zero_grad()\r\n    optimizer_1.step()\r\n    print(\"\u7b2c%d\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a%f\" % (epoch, optimizer_1.param_groups[0]['lr']))\r\n    scheduler_1.step()<\/code><\/pre>\n\n\n\n
        \u521d\u59cb\u5316\u7684\u5b66\u4e60\u7387\uff1a 0.1\r\n\u7b2c1\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.100000\r\n\u7b2c2\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.100000\r\n\u7b2c3\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.100000\r\n\u7b2c4\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.010000\r\n\u7b2c5\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.010000\r\n\u7b2c6\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.010000\r\n\u7b2c7\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.010000\r\n\u7b2c8\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.001000\r\n\u7b2c9\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.001000\r\n\u7b2c10\u4e2aepoch\u7684\u5b66\u4e60\u7387\uff1a0.001000<\/code><\/pre>\n\n\n\n
        \u4e0a\u9762\u7684\u53ea\u662f\u4e00\u90e8\u5206\uff0c\u8fd8\u6709\u5f88\u591a\u66f4\u65b0\u65b9\u6cd5\uff0c\u53ef\u4ee5\u5728\u5b98\u65b9\u6e90\u7801\u4e2d\u67e5\u770b<\/p>\n","protected":false},"excerpt":{"rendered":"
        \u5f53\u6211\u4eec\u5728\u8bad\u7ec3\u795e\u7ecf\u7f51\u7edc\u65f6\u5019\uff0c\u5b66\u4e60\u7387\u4f5c\u4e3a\u4e00\u4e2a\u8d85\u53c2\u6570\uff0c\u9700\u8981\u6211\u4eec\u6307\u5b9a\uff0c\u5982\u679clr\u8fc7\u5927\uff0c\u4f1a\u5bfc\u81f4\u6a21\u578b\u4e0d\u6536\u655b\uff0c\u9707\u8361\uff0c\u800c\u5982\u679clr … \u7ee7\u7eed\u9605\u8bfbtorch.optim.lr_scheduler  \u5b66\u4e60\u7387\u7684\u52a8\u6001\u8c03\u6574<\/span><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[11],"tags":[],"_links":{"self":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts\/3337"}],"collection":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/comments?post=3337"}],"version-history":[{"count":5,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts\/3337\/revisions"}],"predecessor-version":[{"id":3344,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/posts\/3337\/revisions\/3344"}],"wp:attachment":[{"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/media?parent=3337"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/categories?post=3337"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/139.9.1.231\/index.php\/wp-json\/wp\/v2\/tags?post=3337"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}