[SOLVED] deep learning python network cuda GPU

201920201

17 3

1 1
1.1 1
1.2 2
2 3
2.1 4
2.2 5
2.2.15
2.2.15
2.2.2 6
2.2.3 7
2.2.4 8
2.3 8
2.3.1 PyCharm8
2.3.2 PyCharm9
3 9
3.19
3.29
3.2.1 9
3.2.2 10
3.2.3 10
4 10
4.110
4.1.110
4.1.2 Resnet5011
4.212
4.2.112
4.1.2 VGG1613
5 14
5.114
5.215
5.315
6 16
6.116
6.2 17
7 21
7.121
7.222
8 23
24
25

1
1.1

a
b
c
d Convolutional Neural NetworkCNN
Pooling Layer

Nair1retifiedRanzato2Susskind3RBMLuo4DBNlogisticCNNCNNCNN
1.2
CNN
5004001008020Resnet
CKangercontemptdisgustfearhappysadnesssurprise7981VGG19
2
Convolutional Neural Networks, CNNArtificial Neural NetworkANN 212080
1
activation function1

j
h
. . .
. . .
. . .
. . .

j
h
. . .
. . .
. . .
. . .
21.
2.1
Convolutional Neural Networks, CNN
visual perceptiongridlike topology
receptive fieldspatial dimension

22
22LeNetAlexNetVGGNetGoogleNetResnet
2.2
2.2.1
RGB
2.2.1

2
2

2
1
0
2
3
9
5
4
2
0
2
3
4
5
6
1
2
3
1
0
0
4
4
2
8
1
0
1
1
0
1
1
0
1

5
0
1

1
1
5

8
1
3

110012
150412
130415
0
23
233355
2.2.2
pooling
1
1
2
4
5
6
7
8
3
2
1
0
1
2
3
4
6
8
3
4

24
24
2.2.3
fully connected layersFC25
A1
A2
A3

W11
W12
W13
W21
W22
W23
W31
W32
W33
A1
A2
A3

W11
W12
W13
W21
W22
W23
W31
W32
W33
25
3
3
4

2.2.4

4
ReLuReLUReLUReLUsigmoidReLuReLU40CNNNReLU2N
2.3
2.3.1 PyCharm
PyCharmPython IDEPythonProjectIDEDjangoWeb
2.3.2 PyCharm
PyCharmIDE Project
PythonDjango
3
3.1
1.131

31
3.2
3.2.1

1
3.2.2

3.2.3

4
4.1
4.1.1
500re4:14001008020
41 re

bus
dinosaur
elephant
flower
horse
Trainfile
80
80
80
80
80
Testfile
20
20
20
20
20

41 re
4.1.2 Resnet50
Residual Network, ResNet2015ILSVRCGoogLeNetInception v3ResNet50221BNResNet5016

64d
256d
relu
relu
relu
relu
relu

64d
256d
relu
relu
relu
relu
relu

42

ResNet42
42 Resnet50
Layer name
output size
18layer
34layer
50layer
101layer
152layer
conv1
112112
77,64 stride2
conv2.x
5656
33 max pool ,stride2

conv3.x
2828

conv4.x
1414

conv5.x
77

11
Average pool,5d,softmax
FLOPs
1.8109
3.6109
3.8109
7.6109
11.3109
4.2
4.2.1
CKangercontemptdisgustfearhappysadnesssurprise7103
43 CK
anger
contempt
disgust
fear
happy
sadness
surprise
135
54
177
75
207
84
249

43 CK

4.1.2 VGG16
VGGNETVGG16133564128256512
VGG1633

44 VGG16

5

5.1

51
51
5.2

52
5
5.3

53
7
6
6.1
1Cnntest.py

2CnnCK.py

3resnet.py
Resnet50
4vgg.py
VGG16
5windowmain.py

6window1.py

7window2.py

8Recog1.py

6.2
1getvariableGPU
def getvariablex:
xVariablex
return x.cuda if torch.cuda.isavailable else x
2Inference
def inferencemodel, imgpath:
model.eval

imgImage.openimgpath
imgimg.resize224, 224
tensorimgtotensorimg

tensortensor.resize1, 3, 224, 224
tensortensor.cuda GPU

resultmodelgetvariabletensor
resultnpyresult.data.numpy CPUnumpy
maxindexnp.argmaxresultnpy0

return maxindex
3Classifiprob
def Classifiprobmodel, imgpath:
model.eval

imgImage.openimgpath
imgimg.resize224, 224
tensorimgtotensorimg
tensortensor.resize1, 3, 224, 224
resultmodelgetvariabletensor
softmaxtorch.nn.Softmax
resultsoftmaxresult
resultnpyresult.data.numpy
return resultnpy0
4vggclassifi
def vggclassifirawimg:
grayrgb2grayrawimg
grayresizegray, 48, 48, modesymmetric.astypenp.uint8
imggray:, :, np.newaxis
imgnp.concatenateimg, img, img, axis2
imgImage.fromarrayimg
inputstransformtestimg
netvgg.VGGVGG19
checkpointtorch.loados.path.joinmodel, PrivateTestmodel.t7, maplocationcpu
net.loadstatedictcheckpointnet
net.eval
ncrops, c, h, wnp.shapeinputs
inputsinputs.view1, c, h, w
inputsVariableinputs, volatileTrue
outputsnetinputs
outputsavgoutputs.viewncrops, 1.mean0 avg over crops
softmaxtorch.nn.Softmax
scoresoftmaxoutputsavg
scorenpyscore.data.numpy
returnscore
5train
def trainepoch:
printnEpoch: depoch
global Trainacc
net.train
trainloss0
correct0
total0

if epochlearningratedecaystart and learningratedecaystart0:
fracepochlearningratedecaystartlearningratedecayevery
decayfactorlearningratedecayratefrac
currentlropt.lrdecayfactor
utils.setlroptimizer, currentlr set the decayed rate
else:
currentlropt.lr
printlearningrate: sstrcurrentlr

for batchidx, inputs, targets in enumeratetrainloader:
if usecuda:
inputs, targetsinputs.cuda, targets.cuda
optimizer.zerograd
inputs, targetsVariableinputs, Variabletargets
outputsnetinputs
losscriterionoutputs, targets
loss.backward
utils.clipgradientoptimizer, 0.1
optimizer.step

trainlossloss.data0
, predictedtorch.maxoutputs.data, 1
totaltargets.size0
correctpredicted.eqtargets.data.cpu.sum

utils.progressbarbatchidx, lentrainloader, Loss: .3fAcc: .3f dd
trainlossbatchidx1, 100.correcttotal, correct, total

Trainacc100.correcttotal
6test
def testepoch:
global Testacc
global bestTestacc
global bestTestaccepoch
net.eval
PrivateTestloss0
correct0
total0
for batchidx, inputs, targets in enumeratetestloader:
bs, ncrops, c, h, wnp.shapeinputs
inputsinputs.view1, c, h, w

if usecuda:
inputs, targetsinputs.cuda, targets.cuda
inputs, targetsVariableinputs, volatileTrue, Variabletargets
outputsnetinputs
outputsavgoutputs.viewbs, ncrops, 1.mean1 avg over crops
losscriterionoutputsavg, targets
PrivateTestlossloss.data0
, predictedtorch.maxoutputsavg.data, 1
totaltargets.size0
correctpredicted.eqtargets.data.cpu.sum
utils.progressbarbatchidx, lentestloader, Loss: .3fAcc: .3f dd
PrivateTestlossbatchidx1, 100.correcttotal, correct, total
Save checkpoint.
Testacc100.correcttotal
if TestaccbestTestacc:
printSaving..
printbestTestacc: 0.3fTestacc
statenet: net.statedict if usecuda else net,
bestTestacc: Testacc,
bestTestaccepoch: epoch,

if not os.path.isdiropt.datasetopt.model:
os.mkdiropt.datasetopt.model
if not os.path.isdirpath:
os.mkdirpath
torch.savestate, os.path.joinpath, Testmodel.t7
bestTestaccTestacc
bestTestaccepochepoch
7OpenImg
def OpenImg:
global imgpng
global testimage
fnamelfiledialog.askopenfilenametitle, filetypesS2out, .jpg, All Files,
testimagefnamel
ImgImage.openfnamel
ImgImg.resize300, 200, Image.ANTIALIAS
imgpngImageTk.PhotoImageImg
labelImgtk.Labelwindow, imageimgpng
labelImg.placex0, y150
8Resultshow
def Resultshow:
global imgpngc
namelistbus,dinosaur,elephant,flower,horse
numlistClassifiprobresmodel,testimage
colorsBEE7E9, 19caad,718dbf, e84d60, E6CEAC
plt.barhrangelennumlist, numlist, colorcolors, ticklabelnamelist
for a, b in ziprangelennumlist, numlist:
plt.textb0.01, a0.05, :.2f .formatb100, hacenter, vatop, fontsize11
plt.savefigresult.jpg
plt.figure
ImgImage.openresult.jpg
ImgImg.resize550, 350, Image.ANTIALIAS
imgpngcImageTk.PhotoImageImg
labelImgctk.Labelwindow, imageimgpngc
labelImgc.placex300, y50
tk.Labelwindow, width40, text,:.2f.formatnamelistnumlist.tolist.indexmaxnumlist,maxnumlist100,
font, 13.placex500, y450
9Exit
def Exit:
sys.exit

7
7.1

71 a

72 b

7.2

73 a

74 b

8

1 Nair V, Hinton G E. Rectified linear units improve restricted boltzmann machinesC International Conference on International Conference on Machine Learning. Omnipress, 2010:807814.
2 Ranzato M, Susskind J, Mnih V, et al. On deep generative models with applications to recognitionC Computer Vision and Pattern Recognition. IEEE, 2011:28572864.
3 Susskind J, Memisevic R, Hinton G, et al. Modeling the joint density of two images under a variety of transformationsC IEEE Conference on Computer Vision and Pattern Recognition. IEEE Computer Society, 2011:27932800.
4 Luo P. Hierarchical face parsing via deep learningC IEEE Conference on Computer Vision and Pattern Recognition. IEEE Computer Society, 2012:24802487.
5 . CNNJ. . 201604: 871878.
6 . CNNJ. . 201608: 4550.
7 . J. . 201604: 10331038.
8 . J. . 201609: 5557.
9 . J. . 201601: 1622.

Cnntest.py
import torch
from torch.utils.data import DataLoader
from torchvision import datasets,models,transforms
from torch.autograd import Variable
import time

def getvariablex:
xVariablex
return x.cuda if torch.cuda.isavailable else x

datatransformtransforms.Composetransforms.Resize224,224,
transforms.ToTensor,
transforms.Normalizemean0.5,0.5,0.5,
std0.5,0.5,0.5

traindatasetsdatasets.ImageFolderrootC:UsersAdministratorDesktopretrain,
transformdatatransform

testdatasetsdatasets.ImageFolderrootC:UsersAdministratorDesktopretest,
transformdatatransform

traindataloaderDataLoaderdatasettraindatasets,
batchsize20,
shuffleTrue
testdataloaderDataLoaderdatasettestdatasets,
batchsize20,
shuffleTrue
Xexample,Yexamplenextitertraindataloader
exampleclassestraindatasets.classes
indexclassestraindatasets.classtoidx

modelmodels.vgg19pretrainedTrue

Usegputorch.cuda.isavailable

for param in model.parameters:
param.requiresgridFalse

model.fctorch.nn.Linear2048,5

if Usegpu:
modelmodel.cuda

lossftorch.nn.CrossEntropyLoss
optimizertorch.optim.Adammodel.fc.parameters,lr0.00001

epochn50
timeopentime.time

train
for epoch in rangeepochn:
runningloss0.0
runningacc0.0
for i,img, labels in enumeratetraindataloader,1:
imggetvariableimg
labelsgetvariablelabels
optimizer.zerograd
outputsmodelimg
losslossfoutputs, labels
loss.backward
optimizer.step

runninglossloss.item
, predicttorch.maxoutputs, 1
correctnumtorch.sumpredictlabels
runningacccorrectnum.item
runninglosslentraindataloader
runningacclentraindataloader
if i110:
printTrain Epoch:ddIter dd .1fLoss: .5fepoch1, epochn, ilentraindataloader,lentraindataloader.dataset, 100ilentraindataloaderlentraindataloader.dataset,loss.item
timeendtime.timetimeopen
printtimeend
runningloss0.0
runningacc0.0

test
correct0
total0
for j,images, labels in enumeratetestdataloader,1:
imagesgetvariableimages
labelsgetvariablelabels
outputsmodelimages
, predictedtorch.maxoutputs.data, 1
totallabels.size0
correctpredictedlabels.data.sum
printTest Acc:dd dcorrect,total,100correcttotal
timeendtime.timetimeopen
printtimeend

torch.savemodel,C:UsersAdministratorDesktopmodel.pkl

CnnCK.py
from future import printfunction

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.backends.cudnn as cudnn
import torchvision
import transforms as transforms
import numpy as np
import os
import argparse
import utils
from CK import CK
from torch.autograd import Variable
from models import
import vgg

parserargparse.ArgumentParserdescriptionPyTorch CK CNN Training
parser.addargumentmodel, typestr, defaultVGG19, helpCNN architecture
parser.addargumentdataset, typestr, defaultCK, helpdataset
parser.addargumentfold, default1, typeint, helpk fold number
parser.addargumentbs, default128, typeint, helpbatchsize
parser.addargumentlr, default0.01, typefloat, helplearning rate
parser.addargumentresume, r, actionstoretrue, helpresume from checkpoint
optparser.parseargs

usecudatorch.cuda.isavailable

bestTestacc0 best PrivateTest accuracy
bestTestaccepoch0
startepoch0 start from epoch 0 or last checkpoint epoch

learningratedecaystart20 50
learningratedecayevery15
learningratedecayrate0.80.9

cutsize44
totalepoch60

pathos.path.joinopt.datasetopt.model, stropt.fold

Data
print Preparing data..
transformtraintransforms.Compose
transforms.RandomCropcutsize,
transforms.RandomHorizontalFlip,
transforms.ToTensor,

transformtesttransforms.Compose
transforms.TenCropcutsize,
transforms.Lambdalambda crops: torch.stacktransforms.ToTensorcrop for crop in crops,

trainsetCKsplitTraining, foldopt.fold, transformtransformtrain
trainloadertorch.utils.data.DataLoadertrainset, batchsizeopt.bs, shuffleTrue, numworkers1
testsetCKsplitTesting, foldopt.fold, transformtransformtest
testloadertorch.utils.data.DataLoadertestset, batchsize5, shuffleFalse, numworkers1

Model
if opt.modelVGG19:
netvgg.VGGVGG19
elif opt.modelResnet18:
netResNet18

if opt.resume:
Load checkpoint.
print Resuming from checkpoint..
assert os.path.isdirpath, Error: no checkpoint directory found!
checkpointtorch.loados.path.joinpath,Testmodel.t7

net.loadstatedictcheckpointnet
bestTestacccheckpointbestTestacc
bestTestaccepochcheckpointbestTestaccepoch
startepochbestTestaccepoch1
else:
print Building model..

if usecuda:
net.cuda

criterionnn.CrossEntropyLoss
optimizeroptim.SGDnet.parameters, lropt.lr, momentum0.9, weightdecay5e4

Training
def trainepoch:
printnEpoch: depoch
global Trainacc
net.train
trainloss0
correct0
total0

if epochlearningratedecaystart and learningratedecaystart0:
fracepochlearningratedecaystartlearningratedecayevery
decayfactorlearningratedecayratefrac
currentlropt.lrdecayfactor
utils.setlroptimizer, currentlr set the decayed rate
else:
currentlropt.lr
printlearningrate: sstrcurrentlr

for batchidx, inputs, targets in enumeratetrainloader:
if usecuda:
inputs, targetsinputs.cuda, targets.cuda
optimizer.zerograd
inputs, targetsVariableinputs, Variabletargets
outputsnetinputs
losscriterionoutputs, targets
loss.backward
utils.clipgradientoptimizer, 0.1
optimizer.step

trainlossloss.data0
, predictedtorch.maxoutputs.data, 1
totaltargets.size0
correctpredicted.eqtargets.data.cpu.sum

utils.progressbarbatchidx, lentrainloader, Loss: .3fAcc: .3f dd
trainlossbatchidx1, 100.correcttotal, correct, total

Trainacc100.correcttotal

def testepoch:
global Testacc
global bestTestacc
global bestTestaccepoch
net.eval
PrivateTestloss0
correct0
total0
for batchidx, inputs, targets in enumeratetestloader:
bs, ncrops, c, h, wnp.shapeinputs
inputsinputs.view1, c, h, w

if usecuda:
inputs, targetsinputs.cuda, targets.cuda
inputs, targetsVariableinputs, volatileTrue, Variabletargets
outputsnetinputs
outputsavgoutputs.viewbs, ncrops, 1.mean1 avg over crops

losscriterionoutputsavg, targets
PrivateTestlossloss.data0
, predictedtorch.maxoutputsavg.data, 1
totaltargets.size0
correctpredicted.eqtargets.data.cpu.sum

utils.progressbarbatchidx, lentestloader, Loss: .3fAcc: .3f dd
PrivateTestlossbatchidx1, 100.correcttotal, correct, total
Save checkpoint.
Testacc100.correcttotal

if TestaccbestTestacc:
printSaving..
printbestTestacc: 0.3fTestacc
statenet: net.statedict if usecuda else net,
bestTestacc: Testacc,
bestTestaccepoch: epoch,

if not os.path.isdiropt.datasetopt.model:
os.mkdiropt.datasetopt.model
if not os.path.isdirpath:
os.mkdirpath
torch.savestate, os.path.joinpath, Testmodel.t7
bestTestaccTestacc
bestTestaccepochepoch

for epoch in rangestartepoch, totalepoch:
trainepoch
testepoch

printbestTestacc: 0.3fbestTestacc
printbestTestaccepoch: dbestTestaccepoch

resnet.py
import torch
import torch.nn as nn
import torch.nn.functional as F

from torch.autograd import Variable

class BasicBlocknn.Module:
expansion1

def initself, inplanes, planes, stride1:
superBasicBlock, self.init
self.conv1nn.Conv2dinplanes, planes, kernelsize3, stridestride, padding1, biasFalse
self.bn1nn.BatchNorm2dplanes
self.conv2nn.Conv2dplanes, planes, kernelsize3, stride1, padding1, biasFalse
self.bn2nn.BatchNorm2dplanes

self.shortcutnn.Sequential
if stride ! 1 or inplanes ! self.expansionplanes:
self.shortcutnn.Sequential
nn.Conv2dinplanes, self.expansionplanes, kernelsize1, stridestride, biasFalse,
nn.BatchNorm2dself.expansionplanes

def forwardself, x:
outF.reluself.bn1self.conv1x
outself.bn2self.conv2out
outself.shortcutx
outF.reluout
return out

class Bottlenecknn.Module:
expansion4

def initself, inplanes, planes, stride1:
superBottleneck, self.init
self.conv1nn.Conv2dinplanes, planes, kernelsize1, biasFalse
self.bn1nn.BatchNorm2dplanes
self.conv2nn.Conv2dplanes, planes, kernelsize3, stridestride, padding1, biasFalse
self.bn2nn.BatchNorm2dplanes
self.conv3nn.Conv2dplanes, self.expansionplanes, kernelsize1, biasFalse
self.bn3nn.BatchNorm2dself.expansionplanes

self.shortcutnn.Sequential
if stride ! 1 or inplanes ! self.expansionplanes:
self.shortcutnn.Sequential
nn.Conv2dinplanes, self.expansionplanes, kernelsize1, stridestride, biasFalse,
nn.BatchNorm2dself.expansionplanes

def forwardself, x:
outF.reluself.bn1self.conv1x
outF.reluself.bn2self.conv2out
outself.bn3self.conv3out
outself.shortcutx
outF.reluout
return out

class ResNetnn.Module:
def initself, block, numblocks, numclasses7:
superResNet, self.init
self.inplanes64

self.conv1nn.Conv2d3, 64, kernelsize3, stride1, padding1, biasFalse
self.bn1nn.BatchNorm2d64
self.layer1self.makelayerblock, 64, numblocks0, stride1
self.layer2self.makelayerblock, 128, numblocks1, stride2
self.layer3self.makelayerblock, 256, numblocks2, stride2
self.layer4self.makelayerblock, 512, numblocks3, stride2
self.linearnn.Linear512, numclasses

def makelayerself, block, planes, numblocks, stride:
stridesstride1numblocks1
layers
for stride in strides:
layers.appendblockself.inplanes, planes, stride
self.inplanesplanesblock.expansion
return nn.Sequentiallayers

def forwardself, x:
outF.reluself.bn1self.conv1x
outself.layer1out
outself.layer2out
outself.layer3out
outself.layer4out
outF.avgpool2dout, 4
outout.viewout.size0, 1
outF.dropoutout, p0.5, trainingself.training
outself.linearout
return out

def ResNet18:
return ResNetBasicBlock, 2,2,2,2

vgg.py
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable

cfg
VGG11: 64, M, 128, M, 256, 256, M, 512, 512, M, 512, 512, M,
VGG13: 64, 64, M, 128, 128, M, 256, 256, M, 512, 512, M, 512, 512, M,
VGG16: 64, 64, M, 128, 128, M, 256, 256, 256, M, 512, 512, 512, M, 512, 512, 512, M,
VGG19: 64, 64, M, 128, 128, M, 256, 256, 256, 256, M, 512, 512, 512, 512, M, 512, 512, 512, 512, M,

class VGGnn.Module:
def initself, vggname:
superVGG, self.init
self.featuresself.makelayerscfgvggname
self.classifiernn.Linear512, 7

def forwardself, x:
outself.featuresx
outout.viewout.size0, 1
outF.dropoutout, p0.5, trainingself.training
outself.classifierout
return out

def makelayersself, cfg:
layers
inchannels3
for x in cfg:
if xM:
layersnn.MaxPool2dkernelsize2, stride2
else:
layersnn.Conv2dinchannels, x, kernelsize3, padding1,
nn.BatchNorm2dx,
nn.ReLUinplaceTrue
inchannelsx
layersnn.AvgPool2dkernelsize1, stride1
return nn.Sequentiallayers