! les réseaux de neurones artificiels · – +si apprentissage stochastique (après présentation...

!"#$"%"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

90>150(";1*0-+<16)"

9?*1@,*5)A(4B"C"DEF9"""GD9"H%I"

,0>150(:41*0-(<16)J,?*1K,*5)>(4B:L*"

Les Réseaux de Neurones Artificiels""

!"#$"

!"   #$%&'()*+,(%#

-"   #F+)(,-."M"-0("41-4B(#

."   #@(*4(K>*10)"N-6241-4B()#

/"   #'O,KK*(02)),?("/("*(K*+)(0>,210)#

P"

@6,0"

&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

!"#$"Q"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

Introduction

!"#$"

R5161?54,6"E(-*10)"

!"#$"

@1-*S-15"410)5/+*(*"/()"*+)(,-."/("0(-*10()"T"

•  D0)K5*,210"U5161?5S-("–  '("+0'102*#0,>-*(6"V"-0"N1/W6(">*W)")+/-5),0>"

•  ;,K,U6("/O,KK*(02)),?("•  F1U-)>("(>">16+*,0>",-."L,->()"•  XO,441NN1/("/O50L1*N,210)"5041NK6W>()Y"504(*>,50()Y"5NK*+45)()"

•  G,))5Z(N(0>"K,*,66W6("

–  30*'(%04#•  ! "!5!!#%0*'(%04#/,0)"6("+0'102*#6*728%#"

""""8#H$"."%$Q"/,0)"4(6-5"/("!"#$%&!!%="

•  ! !5.# à !5/#+(%%098(%4#:#%0*'(%0#

•  '()%*+%!,-"#.+(*,/,012&(-%,2132#.)#&2%,/,*%42(5)2#*67%8%426,

•  9&:*#4;,%;.&)#)%426,/,&*<&$&)%426,

H"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"

[(*)"/()"*+)(,-."/("0(-*10()",*2345(6)"

•  ;<'28&4#–  ;,64-6)"K,*,66+65),U6()"

–  F1U-)>()"(>">16+*,0>)",-."L,->()"8/5)>*5U-+="

–  96?1*5>BN()")5NK6()"

–  \O(NK615">*W)"?+0+*,6"

•  =>?2*&4#–  ]K,45>+"/()"*,5)100(N(0>)"

–  ^>"/("6OB_K1>BW)("K*1/-5>("

`"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

!"#$"

a5)>1*5S-("8>*W)"65N5>+="

•  @840#0%#>18)0%+0#)04#%0*'(%04#

–  ;,N56651"b16?5"(>"F,N10"c";,<,6"8d"%Ie$="f"6()")_0,K)()"8/,0)"6()",00+()"

%#Q$="

•  @()>A842,(%#?('70AA0#

–  G4";-6614B"g"@5h"8%#iQ="V"%(*"N1/W6("/-"0(-*10("L1*N(6"

–  a(UU"8%#i#="V"*W?6("/O,KK*(02)),?("K,*"*(0L1*4(N(0>"/("41-K6,?("

)_0,K2S-("

e""!"i`"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"I"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

Les réseaux à une couche

!"#$"

a5)>1*5S-("8>*W)"65N5>+="

•  @840#0%#>18)0%+0#)04#%0*'(%04#

–  ;,N56651"b16?5"(>"F,N10"c";,<,6"8d"%Ie$="f"6()")_0,K)()"8/,0)"6()",00+()"%#Q$="

•  @()>A842,(%#?('70AA0#

–  G4";-6614B"g"@5h"8%#iQ="V"%(*"N1/W6("/-"0(-*10("L1*N(6"

–  a(UU"8%#i#="V"*W?6("/O,KK*(02)),?("K,*"*(0L1*4(N(0>"/("41-K6,?(")_0,K2S-("

•  B'0780'4#7()CA04#8%?('72,D*04#

–  9\9'DE^"8j5/*1kla1mY"%#`$="

–  @(*4(K>*10"8F1)(0U6,hY"%#HIl%#`P="

–  90,6_)("K,*"G50)n_"(>"@,K(*>"8%#`#="

#"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"

9*2345,6"E(-*,6"E(>k1*n)"89EE="

X65/("4*(/5>"V"90/*(k"':"E(6)10"

!"#$"

–  F1)(0U6,h"8%#HIl%#`P="

'("@(*4(K>*10"

%%"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

'("K(*4(K>*10"

%P"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

{ biais

x

y

w1w2

w3 w4 w5

w0

wd

1

x1 x2 x3 x4 x5 xd

x0

neurone de biais

1

yi

x(1)

x(2)

x(3)

x(d)

w1i

w2i

w3i

wdi

!(i) =d!

j=0

wjix(j)w0i

!"#$"

'("K(*4(K>*10"V"-0"/5)4*5N50,0>"650+,5*("

%Q"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"

'("K(*4(K>*10"

•  ;EE'0%,442F0#)04#E(8)4#–  @*5045K("82=:!%,-%,>%$$="V"(0"4,)"/(")-44W)Y",<1->(*"M"4B,S-("4100(.510"

S-(6S-("4B1)("/("K*1K1*2100(6"M"6O(0>*+("(>"M"6,")1*2("

FW?6("/-"K(*4(K>*10"V",KK*(0/*(")(-6(N(0>"(0"4,)"/O+4B(4"

%i"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

!"#$"

9KK*(02)),?("/()"K15/)"/()"4100(.510)"

•  ]0"Z(->"N505N5)(*"V"

;,*"01-)"Z1-610)"K1-*">1->()"6()"(.(NK6()"V""

%H"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

wT x ! 0 < 0" # $

% x & '1

'2

" # $

!"#$"

•  Méthode d’exploration de H"

–  Recherche par gradient •  Minimisation de la fonction d’erreur

–  Algorithme :

si la forme est correctement classée : ne rien faire !

sinon : !

boucler sur les formes d’apprentissage jusqu’à critère d’arrêt !

w(t + 1) = w(t) ! " xi ui

w(t + 1) = w(t) ! ! "wE(w(t))

'("K(*4(K>*10"V",6?1*5>BN("/("?*,/5(0>"

%`"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

;,K,45>+"(.K*())5Z("V"X+K,*,210)"650+,5*()"

%e"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"

\5)4*5N50,210"410>*(">1-)"6()",->*()"

;1NN(0>")+K,*(*"K6-)5(-*)"46,))()"

%I"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

!"#$"

\5)4*5N50,210"(0>*("/(-."46,))()"

;1NN(0>")+K,*(*"K6-)5(-*)"46,))()"

%#"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"

'5N5>()"/-"@(*4(K>*10"

•  '("@(*4(K>*10"0("K(->",KK*(0/*("D*0#)04#4>E2'2&'8+04#A8%>28'04"8(:?:"K,)"6("o]F="

•  @,)">1->"M"L,5>"Z*,5")5"10"4B,0?("/O()K,4("/O(0>*+("pG50)n_"(>"@,K(*>Y"%#`#q"

•  G,5)"6("K*+>*,5>(N(0>"()>"/5r456("

•  ^.(NK6("8R5)B1K="

P$"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

!"#$"

–  F1)(0U6,h"8%#HIl%#`P="

'("@(*4(K>*10"

P%"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"

@*(N5(*"4100(.51005)N("V"6("K(*4(K>*10"

•  '2(02&1)16,

–  96?1*5>BN("0%GA8F%0#

–  30#E(*128&#E24#&(*&#2EE'0%)'0"sT"

•  ;,*"0("K(->"K,)">1->"*(K*+)(0>(*"

•  D6"L,->",Z15*"/("U100()"L104210)"/("U,)("8/+>(4>(-*)"614,-.="

•  D6"L,->"),Z15*"6()"41NU50(*"

–  !"#$%&#'(#)*%)+,#-,#$*&.,#

R614,?("

PP"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

!"#$"

a5)>1*5S-("8>*W)"65N5>+="

•  @840#0%#>18)0%+0#)04#%0*'(%04#

–  ;,N56651"b16?5"(>"F,N10"c";,<,6"8d"%Ie$="f"6()")_0,K)()"8/,0)"6()",00+()"%#Q$="

•  @()>A842,(%#?('70AA0#

–  G4";-6614B"g"@5h"8%#iQ="V"%(*"N1/W6("/-"0(-*10("L1*N(6"

–  a(UU"8%#i#="V"*W?6("/O,KK*(02)),?("K,*"*(0L1*4(N(0>"/("41-K6,?(")_0,K2S-("

•  B'0780'4#7()CA04#8%?('72,D*04#

–  9\9'DE^"8j5/*1kla1mY"%#`$="

–  @(*4(K>*10"8F1)(0U6,hY"%#HIl%#`P="

–  90,6_)("K,*"G50)n_"(>"@,K(*>"8%#`#="

•  3(*102*#+(%%098(%%8470#

–  a1K3(6/"8%#IP="V"*+)(,-"U1-46+"

–  @(*4(K>*10"G-62l;1-4B("8%#IH="

–  &"F()(*Z15*"41NK-20?"7"8P$$Q="

–  '()"*+)(,-."/("0(-*10()"K*1L10/)"

–  &"XK5n50?"E(-*10)"7"0(>k1*n)"

PQ""!"

i`"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"Pi"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

Les réseaux multi-couches

!"#$"

E(-*,6"0(>k1*n",KK654,210"

•  9'[DEE"/*5Z()"e$"NKB"10"B5?Bk,_)"

X65/("4*(/5>"V"A:G54B(66"!"#$"

E(-*,6"0(>k1*n",KK654,210"

X65/("4*(/5>"V"90/*,k"E?:"

!"#$"

Composants et structure

du PMC

Pe"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"

•  A1K161?5(">_K5S-("

'("K(*4(K>*10"N-62l41-4B("

PI"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

g(a) = 11 + e! a

yl = g wjk ! jj= 0, d"

#

$ %

&

' ( = g(ak )

Fonction à base radiale

Fonction sigmoïde

Fonction à seuil

Fonction à rampe

Activation ai

Sortie zi

+1

+1

+1

?O8,="t"?8,=8%l?8,=="

'("@(*4(K>*10"G-62l;1-4B()"V"K*1K,?,210"

P#"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

u104210)"/O,42Z,210"

Q$"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

!"#$"

^.(NK6("/("*+)(,-"8)5N-6,>(-*"v,Z,EEX="

Q%"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

L’apprentissage

dans les PMC

QP"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

!"#$"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

9KK*(02)),?(")-K(*Z5)+"

QQ" !"#$"

;,64-6"/("K15/)"

H%&'>0#9!# H%&'>0#9-# I(',0#J#

$" $" $"

$" %" %"

%" $" %"

%" %" $"

Qi"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

;,64-6"/("6,"L104210"o]F"

x2

x1 !"!#

!$

!%

!&

!'

!( !)!*

y

�

i

wi xi

y

!!

"

"

X,-*(wlZ1-)">*1-Z(*"6()"K15/)"?&"T"

!"#$"

;,64-6"/("K15/)"

H%&'>0#9!# H%&'>0#9-# I(',0#J#

$" $" $"

$" %" %"

%" $" %"

%" %" $"

QH"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

;,64-6"/("6,"L104210"o]F"

x2

x1 !"!#

!$

!%

!&

!'

!( !)!*

y

�

i

wi xi

y

!!

"

"

j%"t""%"

jP"t""l%"

jQ"t"l"%"

ji"t""%"

jH"t""%"

j`"t""%"

je"t""$"

jI"t""$"

j#"t""$""

!"#$"

Objectif :

–  Algorithme (rétro-propagation de gradient) : descente de gradient

Algorithme itératif :

Cas hors-ligne (gradient total) :

où :

Cas en-ligne (gradient stochastique) :

w( t ) = w( t!1) ! "# Ew(t )

wij (t) = wij (t !1)!"(t) 1m

#RE (xk ,w)#wijk=1

m

$

wij (t) = wij (t !1)!"(t) #RE(xk,w)#wij

RE(xk,w) = [tk ! f (xk ,w)]2

!

w * = argminw

1m

y(xl ; w) " u(xl )[ ]2

l=1

m

#

'("@(*4(K>*10"G-62l;1-4B()"V",KK*(02)),?("

Q`"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

\()4(0>("/("?*,/5(0>"

Qe"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"

1. Présentation d’un exemple parmi l’ensemble d’apprentissage Séquentielle, aléatoire, en fonction d’un critère donné

2. Calcul de l’état du réseau

3. Calcul de l’erreur = fct(sortie - sortie désirée) (e.g. = (yl - ul)2)

4. Calcul des gradients Par l’algorithme de rétro-propagation de gradient

5. Modification des poids synaptiques

6. Critère d’arrêt Sur l’erreur. Nombre de présentation d’exemples, ...

7. Retour en 1

'("@(*4(K>*10"G-62l;1-4B()"V",KK*(02)),?("

QI"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

1. Evaluation de l’erreur Ej (ou E) due à chaque connexion :

Idée : calculer l’erreur sur la connexion wji en fct de l’erreur après la cellule j

–  Pour les cellules de la couche de sortie :

–  Pour les cellules d’une couche cachée :

! El

! wij

!k = " El

" ak = g' (ak )

" El

" yk = g' (ak ) # uk(xl) $ yk( )

! j = " El

" aj =

" El

" ak

" ak" ajk

# = ! k

" ak" zj

" zj" ajk

# = g' (aj ) $ wjk !kk#

! El

! wij

= ! El

! aj ! aj! wij

= " j zi

@G;"V"6,"*+>*1lK*1K,?,210"/("?*,/5(0>"

Q#"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

",5""V"activation de la cellule i

"w5""V"sortie de la cellule i

"!5""V"erreur attachée à la cellule i

k"5<"j !i ! k ! y"k"

;(66-6("/(")1*2(";(66-6("4,4B+("

!"n"

,"n","<"

!"<"k"<n"

z"j"z"i"

@G;"V"6,"*+>*1lK*1K,?,210"/("?*,/5(0>"

i$"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

•  2. Modification des poids

–  On suppose gradient à pas (constant ou non ): !8>="

–  Si apprentissage stochastique (après présentation de chaque exemple)

–  Si apprentissage total (après présentation de l’ensemble des exemples)

!

"wij = #(t) $ j yi

!

"wij = #(t) $ jn zi

n

n=1

m

%

@G;"V"6,"*+>*1lK*1K,?,210"/("?*,/5(0>"

i%"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

@"V"0(-*10(""/(")1*2("

!"#$"

9#

ai(x) = wjxj

j=1

d

! +w0

yi(x) = g(ai(x))

ys (x) = wjsyjj=1

k

!_)89="

k5)"

n"0(-*10()")-*"6,"

41-4B("4,4B+("

. . .x1 x2 x3 xd

w1 w2 w3wd

yi(x)

x0

w0Biais

. . .y (x)1

'("@G;"V""K,))()",Z,0>"(>",**5W*("8*+)-N+="

iP"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

9#

_)89="

k5)"

:" :" :"."%" ."P" ."Q" ."/"

k"%" k"P" k"Q"k"/"

_"5"8"9"="

."$"

k"$"R"5","5")"

:" :" :"_" 8"9"="%"

!s = g' (as ) (us"ys )

! j = g' (aj ) wjs! scellules scouchesuivante

"

wis(t +1) = wis(t) ! "(t)# sai

wei (t +1) = wei(t) ! "(t )# iae

'("@G;"V""K,))()",Z,0>"(>",**5W*("8*+)-N+="

iQ"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

•  Efficacité en apprentissage

–  En O(w) pour chaque passe d’apprentissage, w = nb de poids

–  Il faut typiquement plusieurs centaines de passes (voir plus loin)

–  Il faut typiquement recommencer plusieurs dizaines de fois un

apprentissage en partant avec différentes initialisations des poids

•  Efficacité en reconnaissance

–  Possibilité de temps réel

@G;"V"',"*+>*1lK*1K,?,210"/("?*,/5(0>"

ii"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"iH"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

D66-)>*,210"

!"#$"i`"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

D66-)>*,210"

!"#$"ie"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

D66-)>*,210"

!"#$"iI"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

D66-)>*,210"

!"#$"

Questions pratiques

i#"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"

x-()210)"

•  "&"R,>4B"7"1-"&"(0l65?0("7"T"

•  "F+?6,?("/-"K,)"/O,KK*(02)),?("–  u5.("

–  [,*5,U6("V"N+>B1/("/-"N1N(0>"

–  G+>B1/()"/("PWN("1*/*("V"4,64-6"/-"a())5(0Y"y"

•  ";*5>W*("/O,**z>"T"

•  "',"&"4,65U*,210"7"/("6,")1*2("

•  ";1/,?("/("6,"41-4B("/(")1*2("T"

•  "x-(66(",*4B5>(4>-*("T"

H$"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

!"#$"

F+?6,?("/-"K,)"/O,KK*(02)),?("

•  A(4B05S-()"

–  9<1->"/O-0"N1N(0>"

–  G+>B1/("/("PWN("1*/*("

H%"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

Apprentissage(s) Reseau non lineaire a une couche

Reseau non lineaire a une couche (1)

Introduction d’une fonction ϕ non lineaire

On a donc Y = ϕ(WX), et les MC ne sont plus applicables

On va appliquer une methode de descente de gradient → rappels !

Algorithme iteratif :

On choisit un Wt=0 aleatoire

Bonne direction = celle ou le critere

baisse

Avancer un peu, mais pas trop

Wt+1 ← Wt − ηdJ (W)

dW

��Wt

avec :

W les parametres ; η : le pas ;dJ (W)

dW

��Wt

: la � bonne �direction

[email protected] () Reseaux de Neurones 9 fevrier 2012 23 / 49

Apprentissage(s) Reseaux multicouches

Reseaux multicouches (6)

Remarques :

On peut sommer les erreurs sur toute la base et retropropager une

seule fois :

→ mode Batch (plus rapide)

Question du pas ... 2eme ordre ? Cf. cours gradient

Quand stopper l’algorithme ? Attention au surapprentissage

[email protected] () Reseaux de Neurones 9 fevrier 2012 33 / 49

!"#$"

',"&"4,65U*,210"7"/("6,")1*2("

•  ;1NN(0>"1U>(05*"-0("K*1U,U565>+"(0")1*2("T"

•  {0(")16-210"V""–  {265),210"/O-0")5?0,6"/O(**(-*"K,*"(0>*1K5("4*15)+("

HP"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

!"#$"

•  1 neurone de sortie

–  {0,1}

–  [0,1] •  Erreur quadratique

–  Probabilité [0,1] •  Critère entropique

|  H907EA0#K#

!  G50()"4_650/*5S-()"!"*14B()"

8http://www.ics.uci.edu/mlearn/MLRepository.html=#

. . .x1 x2 x3 xd

w1 w2 w3wd

x0

w0Biais

. . .

'("41/,?("/("6,"+(*+60#)0#4(',0#

HQ"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

•  c-1 problèmes de discrimination

•  1 neurone de sortie

–  {0,1, …, c}

–  [0,1]

•  n (" c) neurones de sortie

–  1 neurone / classe

–  Code correcteur d’erreur

H907EA0#K#

!  F(4100,5)),04("/("4,*,4>W*()"

N,0-)4*5>)"

!  F(4100,5)),04("/("614->(-*)#

. . .x1 x2 x3 xd

w1 w2 w3wd

yi(x)

x0

w0Biais

. . .y (x)1

9#

JL9M#

'("41/,?("/("6,"+(*+60#)0#4(',0"

Hi"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

ECOC

•  9KK*(0/*("M"/5)20?-(*"P`"6(h*()"

•  ;1/,?(""–  X-*"H"U5>)"T"

–  X-*"%$"U5>)"s"

+/# +0# +1# +2# +3# +4# +5# +6# +7# +/8#9" $" $" $" $" $" $" $" $" $" $"

R" $" $" $" $" %" $" %" $" %" %"

;" $" $" $" %" $" %" $" %" $" %"

i"

`"i"

!"#$"

ECOC

•  ]0",KK*(0/"%$"46,))53(-*)"U50,5*()"

•  ]0"4,64-6(""<A89=Y"<B89=Y"<C89=Y"<D89=Y"<E89=Y"<F89=Y"<G89=Y"<H89=Y"<I89=Y"<AJ89="

•  ]0"/+41/(">89="K,*"K6-)"K*14B("Z15)50"!",-"41/,?("/("9Y"RY";Y"y"

+/# +0# +1# +2# +3# +4# +5# +6# +7# +/8#9" $" $" $" $" $" $" $" $" $" $"

R" $" $" $" $" %" $" %" $" %" %"

;" $" $" $" %" $" %" $" %" $" %"

!"#$"

;*5>W*("/O,**z>"

•  ]K2N,"614,-."

•  \5m+*(04("(0>*("&"U,>4B"7"(>"&"(0l65?0("7"

•  \,0?(*"V"6(")-*l,KK*(02)),?("8&"1Z(*l3}0?"7="

He"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"

Éviter le

Sur-apprentissage

HI"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)="

!"#$"

;1-*U()"/O,KK*(02)),?("(>")-*l,KK*(02)),?("

H#"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

',"*+?*())510"(>"6(")-*l,KK*(02)),?("

`$"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

• "";1-*U()"K1-*"%"$$$"(.(NK6()"

• ,,K4%!!%6,.(42$%6,6&,(*,#L#&),B,JJJ,%;%70!%6,M,

X-*l,KK*(02)),?("8FE="

`%"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

• D>"*(S-5*()"6,U(6(/">*,5050?"/,>,:"96N1)>",66"/,>,"5)"-06,U(6(/: ""

• AB("6(,*050?"2N("/1()"01>")4,6("k(66"

D>"5)"Z(*_")61k"50"0(>k1*n)"k5>B"

N-62K6("B5//(0"6,_(*):"

• D>"4,0"?(>")>-4n"50"K11*"614,6"1K2N,:"

N62&#84#O'(%F#O8&6#P2+QGE'(E2F2,(%R#

R,4n"@*1K,?,210"

• "G-62"6,_(*"@(*4(K>*10"0(>k1*n"4,0"U("

>*,50(/"U_">B("U,4n"K*1K,?,210",6?1*5>BN"

>1"K(*L1*N",0_"N,KK50?"U(>k((0">B("

50K->",0/">B("1->K->:"

;)12%&2F04#

X65/("4*(/5>"V"b(1m*(_"a50>10"

!"#$"

9KK654,210)"

!"#$"

•  F(4100,5)),04("/("L1*N()"

–  D0)K(4210"/("K5W4()"(0"-)50("

–  \5,?01)24"/("/+L,->)"/,0)"/()"N,>+*5,-."8()),5)"010"/()4>*-42L)="

–  F(4100,5)),04("/("4,*,4>W*()"N,0-)4*5>)"

–  D0)K(4210"/(">1N1?*,KB5()"/-")(50"

–  F(4100,5)),04("/("6,"K,*16("

–  95/("M"6,"410/-5>("(0">(NK)"*+(6"

–  :::"

9KK654,210)"/()"*+)(,-."/("0(-*10()"L1*N(6)"

`i"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

9'[DEE"l"9->101N1-)"',0/"[(B546("D0","E(-*,6"E(>k1*n"

9'[DEE"

`H"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

Presented by Martin Ho, Eddy Li, Eric Wong and Kitty Wong - Copyright© 2000

- Developed in 1993.

- Performs driving with Neural Networks.

- An intelligent VLSI image sensor for road following.

- Learns to filter out image details not relevant to driving.

Hidden layer

Output units

Input units

9'[DEE"

``"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

L’apprentissage

de représentations

`e"&"'()"*+)(,-."/("0(-*10()",*2345(6"7"""""""89:";1*0-+<16)=" !"#$"

90,6_)("

F~6("/()"41-4B()"4,4B+()"

!"#$"

F~6("/("6,"41-4B("4,4B+("

`#"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

F~6("/("6,"41-4B("4,4B+("

e$"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

9KK654,210",-."41/()"K1)>,-."8�5K"41/()="

•  [Le Cun et al., 1989, ...] (ATT Bell Labs : très forte équipe)

•  ! 10000 exemples de chiffres manuscrits

•  Segmentés et redimensionnés sur matrice 16 x 16

•  Technique des poids partagés (“weight sharing”)

•  Technique du optimal brain damage

•  99% de reconnaissance correcte (sur l’ensemble d’apprentissage)

•  9% de rejet (pour reconnaissance humaine)

e%"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

',"U,)("/("/100+()"

eP"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

F+)(,-."M"410Z16-210"V"9KK654,210",-."41/()"K1)>,-.""

eQ"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

^.(NK6()"/O(**(-*)"/("K*+/54210"

ei"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

F+)(,-"4100(.51005)>("K1-*"6,"*(4100,5)),04("/("4B5m*()"N,0-)4*5>)"

eH"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

9KK654,210",-."41/()"K1)>,-."8�5K"41/()="V"'(E(>lH"

e`"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

'()"&"/((K"U(65(L"0(>k1*n)"7"

•  G12Z,210"

–  \5)K1)(*"/O-0"()K,4("/OB_K1>BW)()",/,K>+"V"6,0?,?("/("4104(K>)"

–  'O,KK*(0/*("

•  G1_(0"K*1K1)+"

–  F+)(,-."/("0(-*10()"&"K*1L10/)"7"

–  @(*N(h,0>"/(")>+(7E(40'"(0"/()4*5K>(-*)""

–  9Z(4"8%&0')>E0%)2%+04#41NK6(.()"

•  9KK*14B()"–  F+)(,-."M"410Z16-210"8'(;-0"(>",6:="

–  &"\((K"U(65(L"0(>k1*n)"7"8a50>10"(>",6:="

ee"&"D0>*1/-4210"M"6O9KK*(02)),?("9*2345(6"7"""""""89:";1*0-+<16)="""""""""""""^@9Al"P$%i" !"#$"

'()"&"/((K"U(65(L)"0(>k1*n)"7"

•  ^0"-0">*,0)K,*(0>"

eI"&"D0>*1/-4210"M"6O9KK*(02)),?("9*2345(6"7"""""""89:";1*0-+<16)="""""""""""""^@9Al"P$%i"

!"#$"

jB_"\((K"N-62l6,_(*(/"0(-*,6"0(>k1*n"

pixels"

edges"

object parts"(combination "of edges)"

object models"

!"#$"

9KK*(02)),?("/("*(K*+)(0>,210)"B5+*,*4B5S-()"

•  9KK*(02)),?("/("*(K*+)(0>,210)"B5+*,*4B5S-()"

I$"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

#4 Learning multiple levels of representation !"##$%%&'$()*+$,(,-.$/%(,$-/0(+$$1$/(&02$/)$+&-2$(/$1/$%$02-3*0%(

(

4-.$/(5(

4-.$/(6(

4-.$/(7(8&9:;,$'$,(

,&09"&%3#(/$1/$%$02-3*0%(

<4$$=(>:-)=(4-/9)-0(?(@9=(@A>!(6BBCD(<4$$=(E/*%%$=(F-09-0-2:(?(@9=(AGH4(6BBCD((

6I(

!"#$"%&'()*"+,#(-&./01$"2&3&0$(0*412&0$54/01+,&*64"*22*)&78&5'+94+*&+*:*+2&$.&/;21"/09$(&(

!/"12&0$5;#(*&1$&.$"5&$;<*012&

!"#$"

9KK*(02)),?("/("*(K*+)(0>,210)"B5+*,*4B5S-()"

I%"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$%&'()"*+',+-.

!""#$%"&'()(*$+$%"&'()(*$,)"'&'-.)-&/$0"#'"1"(2&2)3(1

4251$6""#$)7$)2$.&1$83'"$2.&($3("$12&*"$37$(3(9/)("&'$7"&2:'"$2'&(173'8&2)3(

/0+1'+23$"%3+44151$0

#.67#$8$39$+-&0$

(1:7#$8$39$+-&0$

;1<=7#$8$39$+-&0$

9$+-&0$"814&+31,+-1.'".5">.'8.3&-1.'+3"'$-"-0+1'$:".'"?@+<$A$-"50.@"BC$13$0"D"9$0<&4"EFGHI!"#$"

9KK*(02)),?("

•  9KK*(02)),?(")-K(*Z5)+"30,6"

IP"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!

!

input

features

! More abstract features

! Even more abstract

features

Output f(X) six

Target Y

two! = ?

Supervised Fine-Tuning

•  !""#$%&'()*+,%-*./#/0)1.'-23./)4%%")1%3)5678)4%%")1%3)56+978):;)

!"#$"

D66-)>*,210"

•  X_)>WN("/+Z(61KK+"K,*"b11?6("(>"{:"/("X>,0L1*/"

•  F(4100,5)),04("/("Z5),?()"

–  X1-)"410/5210)"/("6-N5W*("/5Z(*)()"

–  X1-)">1->",0?6("

•  9KK*(02)),?("010")-K(*Z5)+"

–  #"41-4B()"f"%$Ä#"4100(.510)"

–  %$"N566510)"/O5N,?()"

–  Q"<1-*)"/("4,64-6")-*"%`"$$$"K*14())(-*)"

•  9N+651*,210"/()"K(*L1*N,04()"/("e$Å"!"+>,>"/("6O,*>"

IQ"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

'()"&"/((K"U(65(L"0(>k1*n)"7"

Ii"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

'()"&"/((K"U(65(L"0(>k1*n)"7"

IH"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)" !"#$"

{0("5/+("50>*5?,0>("V"6("&"*()(*Z15*"41NK-20?"7"

•  D/+("V"–  {265)(*"-0"*+)(,-"*+4-**(0>"),0)"6O(0>*,Ç0(*"(.K6545>(N(0>"

–  ^0>*,Ç0(*"-0(")(-6("41-4B("/(")1*2("

|  F+l50>*1/-5>"-0("&"/_0,N5S-("7"

!  X+*5()">(NK1*(66()"

I`"&"D0>*1/-4210"M"6O9KK*(02)),?("9*2345(6"7"""""""89:";1*0-+<16)="""""""""""""^@9Al"P$%i"

!"#$"

@*+/54210"V"&"*()(*Z15*"41NK-20?"7"

Ie"j1*nX>,>lP$%Q"""&"'O9KK*(02)),?("9*2345(6"L,4(",-."/100+()">(NK1*(66()"7""""""89:";1*0-+<16)=""

output neuronactivation y n( )

teacherd n( )

wi

"reservoir" with three traces x ni( )

B

A

!

!"#$%&$%!'"#$%&'()*(+,#+-*.-*#+/#*0&#*%1(,&'#234#!'+**&'"#+5&%61('#7(*0#*0&#)+%%&)*#)+,*(,-1*(+,#!8+6('"9#!("#:&1%,(,;#*0&#<=#1**%1)*+%9#>0%&&#81?.6&#1)*(51*(+,#*%1)&8#+/#(,*&%,16#,&-%+,8#1%&#80+7,9#>0&@#A&)0+A#*0&#*&1)0&%#8(;,16#!""#9#B/*&%#*%1(,(,;C#*0&#'&8(%&'#+-*.-*#(8#%&D)%&1*&'#/%+?#*0&#&)0+#8(;,168#*0%+-;0#+-*.-*#)+,,&)*(+,8#!'+**&'#1%%+78"#70+8&#7&(;0*8##$#1%&#*0&#%&8-6*#+/#*0&#*%1(,(,;#.%+)&'-%&9#!!$%&'(!)*!+&*,(*-!,!&,.-/0!122!)%(3!4555!.*6&/.'!"+,77*-!(3*!8&*'*&9/%&8#!,.-!/.*!/6(:6(!.*6&/.;!<0:/&(,.(7=>!(3*!/6(:6(!.*6&/.!),'!*?6%::*-!)%(3!&,.-/0!+/..*+(%/.'!(3,(!:&/@*+(!A,+B!%.(/!(3*!&*'*&9/%&!"$%C;!DE#;!F!G555!'(*:!(*,+3*&!'*?6*.+*!!"4#>;;;>!!"G555#%),'!C*.*&,(*-!H&/0!(3*!IJK!*?6,(%/.!,.-!H*-!%.(/!(3*!/6(:6(!.*6&/.;!L3%'!*M+%(*-!(3*!%.(*&.,7!.*6&/.'!(3&/6C3!(3*!/6(:6(!H**-A,+B!+/..*+(%/.';!FH(*&!,.!%.%(%,7!(&,.'%*.(>!(3*=!'(,&(*-!(/!*M3%A%(!'='(*0,(%+!%.-%9%-6,7!9,&%,(%/.'!/H!(3*!(*,+3*&!'*?6*.+*!"$%C;!DE#;!!!L3*!H,+(!(3,(!(3*!%.(*&.,7!.*6&/.'!-%':7,=!'='(*0,(%+!9,&%,.('!/H!(3*!*M+%(%.C!*M(*&.,7!'%C.,7!%'!+/.'(%(6(%/.,7!H/&!NK2'O!(3*!%.(*&.,7!.*6&/.'!06'(!)/&B!,'!8*+3/!H6.+(%/.'8!H/&!(3*!-&%9%.C!'%C.,7;!2/(!*9*&=!&,.-/07=!C*.*&,(*-!122!3,'!(3%'!:&/:*&(=>!A6(!%(!+,.!*HH*+(%9*7=!A*!A6%7(!%.(/!,!&*'*&9/%&!"'6::/&(%.C!/.7%.*!(*M(#;!!!<(!%'!%0:/&(,.(!(3,(!(3*!8*+3/8!'%C.,7'!A*!&%+37=!9,&%*-;!L3%'!),'!*.'6&*-!A=!,!':,&'*!%.(*&+/..*+(%9%(=!/H!4P!)%(3%.!(3*!&*'*&9/%&O!(3%'!+/.-%(%/.!7*('!(3*!&*'*&9/%&!-*+/0:/'*!%.(/!0,.=!7//'*7=!+/6:7*-!'6A'='(*0'>!*'(,A7%'3%.C!,!&%+37=!'(&6+(6&*-!&*'*&9/%&!/H!*M+%(,A7*!-=.,0%+';!!!FH(*&!(%0*!"!Q!G555>!/6(:6(!+/..*+(%/.!)*%C3('!#$!"$!Q!4>!;;;>!4555#!)*&*!+/0:6(*-!"-,'3*-!,&&/)'!%.!$%C;!4E#!H&/0!(3*!7,'(!D555!'(*:'!"%&%4554>;;;>!G555!/H!(3*!(&,%.%.C!&6.%'6+3!(3,(!(3*!

(&,%.%.C!*&&/&! !),'!0%.%0%R*-!S'!! !! !

!"

G555

4554

D4555

4#"#"D555T4IKN

" $$ $()*$" "'#"! " $""#O!,+(%9,(%/.!/H!(3*!$U(3!%.(*&.,7!.*6&/.!,(!(%0*!"V;!L3%'!%'!,!'%0:7*!7%.*,&!&*C&*''%/.;!!!

! G


teacherd n( )

wi


B

A

!


(&,%.%.C!*&&/&! !),'!0%.%0%R*-!S'!! !! !

!"

G555

4554

D4555

4#"#"D555T4IKN

" $$ $()*$" "'#"! " $""#O!,+(%9,(%/.!/H!(3*!$U(3!%.(*&.,7!.*6&/.!,(!(%0*!"V;!L3%'!%'!,!'%0:7*!7%.*,&!&*C&*''%/.;!!!

! G


teacherd n( )

wi


B

A

!


(&,%.%.C!*&&/&! !),'!0%.%0%R*-!S'!! !! !

!"

G555

4554

D4555

4#"#"D555T4IKN

" $$ $()*$" "'#"! " $""#O!,+(%9,(%/.!/H!(3*!$U(3!%.(*&.,7!.*6&/.!,(!(%0*!"V;!L3%'!%'!,!'%0:7*!7%.*,&!&*C&*''%/.;!!!

! G


teacherd n( )

wi


B

A

!


(&,%.%.C!*&&/&! !),'!0%.%0%R*-!S'!! !! !

!"

G555

4554

D4555

4#"#"D555T4IKN

" $$ $()*$" "'#"! " $""#O!,+(%9,(%/.!/H!(3*!$U(3!%.(*&.,7!.*6&/.!,(!(%0*!"V;!L3%'!%'!,!'%0:7*!7%.*,&!&*C&*''%/.;!!!

! G

&"^4B1)"7"

X5?0,6"M"&",KK*(0/*("7"


teacherd n( )

wi


B

A

!


(&,%.%.C!*&&/&! !),'!0%.%0%R*-!S'!! !! !

!"

G555

4554

D4555

4#"#"D555T4IKN

" $$ $()*$" "'#"! " $""#O!,+(%9,(%/.!/H!(3*!$U(3!%.(*&.,7!.*6&/.!,(!(%0*!"V;!L3%'!%'!,!'%0:7*!7%.*,&!&*C&*''%/.;!!!

! G

X1*2("

/+)5*+("

!"#$"

'()"*+)(,-."M"0(-*10()"5NK-6)5100(6)"

•  90&N&*:,*%42#!,*%)?(2N6,

II"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

9"*(>(05*"

%:  G+>B1/("/O50/-4210")-K(*Z5)+(">*W)"Z(*),26("

–  A*W)"?*,0/"4B,NK"/O,KK654,210:"u,N566("/("N1/W6()">*W)")1-K6(:"

–  ;(*>,50(N(0>"K,)"-0"N1/W6("U5161?5S-("

P:  9KK*(02)),?("/(N,0/,0>"/-")150"

•  ;B15."/("6O,*4B5>(4>-*("

•  9KK*(02)),?("6(0>Y")-<(>"M"N505N,"614,-."

Q:  &"]K,45>+"7"

–  \5r456("/O50>(*K*+>(*"6()"*+)(,-.",KK*5)"

–  \5r456("/("N(h*("/("6,"4100,5)),04("#,02&(2&"

I#"G,)>(*"DXD"!";1-*)"DXDlQ""C""F+)(,-."/("0(-*10()",*2345(6)"

!"#$"

Références bibliographiques

•  90>150(";]FE{Év]'X"(>"',-*(0>"GD;'^A"

O002%*+66#:%,#2+P.&%!Q,R(*.%0)6,%),#!:(2&)<7%6"

^_*166()Y"P$%$"

•  b:"\*(_L-)"(>",6:"

O002%*+66#:%,6)#+6+S4%Q,T16%#4;,-%,*%42(*%6Q,R#2)%6,)(0(!(:&S4%6Q,U#.<&*%6,V,

L%.)%426,6400(2)"

^_*166()Y"P$$I:"8G,6?*+"6("2>*("?+0+*,6Y"Z*,5N(0>"/+/5+",-."FE)="

•  X5N10"a9cÑDE"

W%42#!,W%)?(2N6Q,O,.(702%<%*6&L%,3(4*-#+(*"

@*(024("a,66Y"%###"

•  X5N10"a9cÑDE"

W%42#!,W%)?(2N6,#*-,X%#2*&*:,U#.<&*%6"

@*(024("a,66Y"P$$#:"8F(,66_"/5m(*(0>"L*1N">B("%###O)"10(="

! les réseaux de neurones artificiels · – +si apprentissage stochastique (après présentation...

Documents