0\64/$GPLQ &RRNERRN
JUHDWUHFLSHVIRUPDVWHULQJ0\64/FRQÀJXUDWLRQDQG DGPLQLVWUDWLRQ
ĂŶŝĞů^ĐŚŶĞůůĞƌ hĚŽ^ĐŚǁĞĚƚ
%,50,1*+$0080%$,
0\64/$GPLQ&RRNERRN &RS\ULJKW3DFNW3XEOLVKLQJ
$OOULJKWVUHVHUYHG1RSDUWRIWKLVERRNPD\EHUHSURGXFHGVWRUHGLQDUHWULHYDOV\VWHPRU WUDQVPLWWHGLQDQ\IRUPRUE\DQ\PHDQVZLWKRXWWKHSULRUZULWWHQSHUPLVVLRQRIWKHSXEOLVKHU H[FHSWLQWKHFDVHRIEULHITXRWDWLRQVHPEHGGHGLQFULWLFDODUWLFOHVRUUHYLHZV (YHU\HIIRUWKDVEHHQPDGHLQWKHSUHSDUDWLRQRIWKLVERRNWRHQVXUHWKHDFFXUDF\RIWKH LQIRUPDWLRQSUHVHQWHG+RZHYHUWKHLQIRUPDWLRQFRQWDLQHGLQWKLVERRNLVVROGZLWKRXW ZDUUDQW\HLWKHUH[SUHVVRULPSOLHG1HLWKHUWKHDXWKRUVQRU3DFNW3XEOLVKLQJDQGLWVGHDOHUV DQGGLVWULEXWRUVZLOOEHKHOGOLDEOHIRUDQ\GDPDJHVFDXVHGRUDOOHJHGWREHFDXVHGGLUHFWO\RU LQGLUHFWO\E\WKLVERRN 3DFNW3XEOLVKLQJKDVHQGHDYRUHGWRSURYLGHWUDGHPDUNLQIRUPDWLRQDERXWDOORIWKHFRPSDQLHV DQGSURGXFWVPHQWLRQHGLQWKLVERRNE\WKHDSSURSULDWHXVHRIFDSLWDOV+RZHYHU3DFNW 3XEOLVKLQJFDQQRWJXDUDQWHHWKHDFFXUDF\RIWKLVLQIRUPDWLRQ
)LUVWSXEOLVKHG 0DUFK
3URGXFWLRQ5HIHUHQFH
3XEOLVKHGE\3DFNW3XEOLVKLQJ/WG /LQFROQ5RDG 2OWRQ %LUPLQJKDP%3$8. ,6%1 ZZZSDFNWSXEFRP &RYHU,PDJHE\9LQD\DN&KLWWDU
[email protected]
&UHGLWV $XWKRUV 'DQLHO6FKQHOOHU
(GLWRULDO7HDP/HDGHU $NVKDUD$ZDUH
8GR6FKZHGW ,QGH[HU 5HYLHZHUV
5HNKD1DLU
.DL6HLGOHU 0DUF'HOLVOH
3URMHFW7HDP/HDGHU /DWD%DVDQWDQL
$FTXLVLWLRQ(GLWRU 6DUDK&XOOLQJWRQ
3URMHFW&RRUGLQDWRU 6KXEKDQMDQ&KDWWHUMHH
'HYHORSPHQW(GLWRU 5HVKPD6XQGDUHVDQ
3URRIUHDGHU &KULV6PLWK
7HFKQLFDO(GLWRUV 3DOODYL.DFKDUH %KXSDOL.KXOH $DURQ5RVDULR &RS\(GLWRU /DNVKPL0HQRQ
*UDSKLFV *HHWDQMDOL6DZDQW 3URGXFWLRQ&RRUGLQDWRU 0HOZ\Q$UXQ' VD &RYHU:RUN 0HOZ\Q$UXQ' VD
$ERXWWKH$XWKRUV 'DQLHO6FKQHOOHUZRUNVDVDVRIWZDUHGHYHORSHUGDWDEDVHDGPLQLVWUDWRUDQGJHQHUDO ,7SURIHVVLRQDOIRUDQLQGHSHQGHQWVRIWZDUHYHQGRULQWKHUHWDLOVHFWRU$IWHUVXFFHVVIXOO\ JUDGXDWLQJIURPWKH8QLYHUVLW\RI&RRSHUDWLYH(GXFDWLRQLQ+HLGHQKHLP*HUPDQ\ZLWKD GHJUHHLQ%XVLQHVV&RPSXWHU6FLHQFHKHVWDUWHGKLVFDUHHUDVDSURIHVVLRQDOVRIWZDUH GHYHORSHUIRFXVHGRQWKH0LFURVRIWWHFKQRORJ\VWDFN,QKHVWDUWHGIRFXVLQJRQ HQWHUSULVHOHYHO-DYDGHYHORSPHQWDQGKDVVLQFHJDLQHGH[WHQVLYHNQRZOHGJHDQGH[SHULHQFH LPSOHPHQWLQJODUJHVFDOHV\VWHPVEDVHGRQ-DYD((DQGUHODWLRQDOGDWDEDVHVHVSHFLDOO\ 0\64/VLQFHYHUVLRQ &XUUHQWO\KHLVPRVWO\LQYROYHGZLWKWKHRQJRLQJGHYHORSPHQWRIIUDPHZRUNOHYHO IXQFWLRQDOLW\LQFOXGLQJFXVWRPL]DWLRQDQGH[WHQVLRQRIDQ250EDVHGSHUVLVWHQFHOD\HU +HLVLQYROYHGLQGLIIHUHQWRSHQVRXUFHSURMHFWVVXFKDV)LQG%XJV(FOLSVHDQG&KHFNVW\OH DQGLQIUHTXHQWO\EORJVDERXW-DYD0\64/:LQGRZV/LQX[DQGRWKHULQVDQLWLHVDW http://www.danielschneller.com tŚĞŶ/ĮƌƐƚǁĂƐĂƐŬĞĚďLJWĂĐŬƚWƵďůŝƐŚŝŶŐǁŚĞƚŚĞƌ/ǁŽƵůĚďĞŝŶƚĞƌĞƐƚĞĚŝŶ ǁƌŝƟŶŐĂŬĂďŽƵƚDLJ^Y>ŽŶŚƌŝƐƚŵĂƐ ǀĞϮϬϬϴůŝƩůĞĚŝĚ/ŬŶŽǁŚŽǁŵƵĐŚ ǁŽƌŬ͕ƐƚƌĞƐƐ͕ďƵƚĂůƐŽǁŚĂƚĂůŽƚŽĨĨƵŶ/ǁĂƐŚĞĂĚĞĚĨŽƌ͘ EŽǁ͕ƚŚĂƚƚŚĞŬŝƐĮŶĂůůLJĚŽŶĞ/ǁŽƵůĚůŝŬĞƚŽƚŚĂŶŬƚŚŽƐĞƉĞŽƉůĞǁŝƚŚŽƵƚ ǁŚŽŵŐĞƫŶŐŝƚĚŽŶĞǁŽƵůĚŚĂǀĞďĞĞŶŝŵƉŽƐƐŝďůĞ͘ &ŝƌƐƚŽĨĂůů͕/ΖĚůŝŬĞƚŽƚŚĂŶŬhĚŽĨŽƌĂŐƌĞĞŝŶŐƚŽďĞŵLJĐŽͲĂƵƚŚŽƌ͘tŝƚŚŽƵƚŚŝŵ͕ ƚŚŝƐǁŚŽůĞƚŚŝŶŐǁŽƵůĚŚĂǀĞƚĂŬĞŶĂůŽƚůŽŶŐĞƌĂŶĚǁŽƵůĚŚĂǀĞďĞĞŶŶŽƚŚĂůĨĂƐ ƵƐĞĨƵůĂƐ/ďĞůŝĞǀĞŝƚŚĂƐƚƵƌŶĞĚŽƵƚŶŽǁ͘ /ǁŽƵůĚĂůƐŽůŝŬĞƚŽƚŚĂŶŬƚŚĞƚĞĂŵĂƚWĂĐŬƚWƵďůŝƐŚŝŶŐͶŵŽƐƚŝŵƉŽƌƚĂŶƚůLJ ĨŽƌŶŽƟĐŝŶŐĂŶĚƌĞĂĚŝŶŐŵLJďůŽŐ͕ĐŽŶƐĞƋƵĞŶƚůLJĐŽŶƚĂĐƟŶŐŵĞƚŽŐĞƚƚŚĞ ǁŚŽůĞƚŚŝŶŐƐƚĂƌƚĞĚͶďƵƚĂůƐŽĨŽƌƚĂŬŝŶŐĐĂƌĞŽĨƐĐŚĞĚƵůĞƐ͕ƉƌŽǀŝĚŝŶŐƐƵƉƉŽƌƚ͕ ŐƵŝĚĂŶĐĞĂŶĚĨĞĞĚďĂĐŬ͕ĂŶĚŬĞĞƉŝŶŐƵƐŽŶƚƌĂĐŬƚŚĞǁŚŽůĞǁĂLJ͘ >ĂƐƚ͕ďƵƚďLJŶŽŵĞĂŶƐůĞĂƐƚ͕/ǁĂŶƚƚŽƚŚĂŶŬ:ĞŶŶLJͶĨŽƌĞŶĐŽƵƌĂŐŝŶŐŵĞƚŽ ǁƌŝƚĞĂŬŝŶƚŚĞĮƌƐƚƉůĂĐĞ͕ĂŶĚƚŚĞŶŵĂŬŝŶŐƐƵƌĞ/ŶĞǀĞƌƌĂŶŽƵƚŽĨƚĞĂ͕ ĐŽŽŬŝĞƐ͕ŽƌŵŽƟǀĂƟŽŶŽŶƚŚĞĐŽƵŶƚůĞƐƐĞǀĞŶŝŶŐƐ/ƐƉĞŶƚƐŝƫŶŐŝŶĨƌŽŶƚŽĨƚŚĞ ŬĞLJďŽĂƌĚŝŶƐƚĞĂĚŽĨǁŝƚŚŚĞƌ͘/ĚĞĚŝĐĂƚĞƚŚŝƐŬƚŽŚĞƌ͘
8GR6FKZHGWKDVRYHUWHQ\HDUVRIH[SHULHQFHLQWKH,7LQGXVWU\DVDSURIHVVLRQDO-DYDGH YHORSHUDQGVRIWZDUHDUFKLWHFW+HLVKHDGRIWKH-DYDDUFKLWHFWXUHWHDPDQGGHSXW\KHDGRI WKH-DYDGHYHORSPHQWGHSDUWPHQWDWWKH,7VHUYLFHSURYLGHUIRU*HUPDQ\ VPDUNHWOHDGHULQ WKH'R,W START SLAVE;
:KHQZHZLVKWRGUDZ\RXUDWWHQWLRQWRDSDUWLFXODUSDUWRIDFRGHEORFNWKHUHOHYDQWOLQHVRU LWHPVDUHVHWLQEROG slave> SHOW SLAVE STATUS\G ************************** 1. row *************************** ... Slave_IO_Running: Yes Slave_SQL_Running: Yes ...
1HZWHUPVDQGLPSRUWDQWZRUGVDUHVKRZQLQEROG:RUGVWKDW\RXVHHRQWKHVFUHHQLQ PHQXVRUGLDORJER[HVIRUH[DPSOHDSSHDULQWKHWH[WOLNHWKLV -> ->
CREATE DATABASE repdb; USE repdb; CREATE TABLE tblA ( id INT(10) PRIMARY KEY NOT NULL, label VARCHAR(30) ) ENGINE=InnoDB; CREATE TABLE tblB ( name VARCHAR(20) PRIMARY KEY NOT NULL, age INT(3) ) ENGINE=InnoDB;
&KDSWHU master> INSERT INTO tblA VALUES -> (1, 'label 1'), -> (2, 'label 2'), -> (3, 'label 3'); master> INSERT INTO tblB VALUES -> ('Peter', 55), -> ('Paul', 43), -> ('Mary', 25);
,QVHUWLQJWKLVGDWDDOUHDG\FUHDWHVELQORJLQIRUPDWLRQ GRANT REPLICATION SLAVE -> ON *.* -> TO 'repblack'@'localhost' -> IDENTIFIED BY 'blackpass';
&RQÀJXUHWKHÀOWHU EODFNKROH LQVWDQFHZLWKDFRQÀJXUDWLRQÀOHQDPHGmy.black WKDW FRQWDLQVDWOHDVWWKHIROORZLQJ [client] port
= 3308
socket
= /home/ds/blacktest/black.sock
[mysqld_safe] socket
= /home/ds/blacktest/black.sock
[mysqld] log-slave-updates skip-innodb default-storage-engine=blackhole user pid-file socket port basedir
= = = = =
mysql /home/ds/blacktest/black.pid /home/ds/blacktest/black.sock 3308 /home/ds/blacktest
5HSOLFDWLRQ datadir tmpdir language bind-address server-id log-bin relay-log
= /home/ds/blacktest/data.black = /tmp = /home/ds/blacktest/share/mysql/english = 127.0.0.1 = 2 = /home/ds/blacktest/black-bin.log = /home/ds/blacktest/black-relay.log
1RWLFHWKDWDOORFFXUUHQFHVRIPDVWHUKDYHEHHQUHSODFHGZLWKEODFN 0RUHRYHUWKHserver-idVHWWLQJKDVEHHQFKDQJHGDQGWKH log-slave-updatesskip-innodbDQGdefault-storage-engine RSWLRQVKDYHEHHQDGGHG7KHVHFRQGRQHSUHYHQWVWKLVLQVWDQFHIURP FUHDWLQJibdataWDEOHVSDFHÀOHVZKLFKZRXOGQRWEHXVHGODWHUDQ\ZD\ 7KHODVWRQHVSHFLÀHVZKLFKVWRUDJHHQJLQHWRXVHZKHQDCREATETABLE VWDWHPHQWGRHVQRWH[SOLFLWO\VSHFLI\RQHRULIWKHVSHFLÀHGHQJLQHLVQRW DYDLODEOH:HZLOOFRPHEDFNWRWKLVVRRQ
0DNHVXUHWKLVLQVWDQFHEDVLFDOO\ZRUNVE\VWDUWLQJLWWKHVDPHZD\DVWKHPDVWHU EHIRUH \RXZLOOQRWVHHWKH,QQR'%PHVVDJHVKHUHRIFRXUVH ~/blacktest$ xterm -T BLACK -e bin/mysqld \ > --defaults-file=my.black \ > --console &
&UHDWHDVHWRIGXPSÀOHVIURPWKHPDVWHU WRVHWXSERWKWKHEODFNKROHÀOWHUDQGDQH[DPSOHVODYH7KHGHWDLOVRQZK\ZHQHHG WZRDQGLQZKLFKZD\VWKH\DUHGLIIHUHQWZLOOEHH[SODLQHGODWHU8VHWKHVHFRPPDQGV WRFUHDWHWKHÀOHVQHHGHG ~/blacktest$ bin/mysqldump -S master.sock -uroot \ > --master-data \ > --single-transaction \ > --no-create-info \ > --ignore-table=repdb.tblA \ > repdb > master_data.sql ~/blacktest$ bin/mysqldump -S master.sock -uroot \ > --no-data \ > repdb > master_struct.sql
&KDSWHU
&RQQHFWWRWKHÀOWHUVHUYHUFUHDWHWKHGDWDEDVHPDNHLWWKHGHIDXOWGDWDEDVHDQG ÀQDOO\LPSRUWWKHVWUXFWXUHLQIRUPDWLRQFUHDWHGEHIRUH ~/blacktest$ bin/mysql -uroot -S black.sock --prompt='black> ' black> CREATE DATABASE repdb; black> USE repdb; black> source master_black.sql;
$WWKLVSRLQWZHQRZKDYHWKHVWUXFWXUHRIWKHPDVWHUWUDQVIHUUHGWRWKHÀOWHUHQJLQH DGDSWHGWRXVHWKHEODFNKROHHQJLQHIRUDOOWKHWDEOHV
6HWXSWKHUHSOLFDWLRQEHWZHHQPDVWHUDQGÀOWHUHQJLQH7RGRVRZH QHHGWRNQRZWKHH[DFWSRVLWLRQIURPZKHUHWKHÀOWHUZLOOVWDUWUHSOLFDWLQJ([WUDFWWKLV LQIRUPDWLRQIURPWKHSUHYLRXVO\WDNHQGDWDGXPSOLNHWKLV
~/blacktest$ head -n 30 master_data.sql | grep 'CHANGE MASTER TO'
:ULWHGRZQWKDWLQIRUPDWLRQ ZHZLOOQHHGLWLQDPRPHQW
0RGLI\WKHmy.blackFRQÀJXUDWLRQÀOHWRFRQWDLQWKHIROORZLQJLQWKH[mysqld] VHFWLRQ replicate-ignore-table=repdb.tblA replicate-do-table=repdb.tblB
7KLVLVDYHU\VLPSOHÀOWHUVHWXS LQDUHDODSSOLFDWLRQVFHQDULRWKHVHUXOHVZLOO SUREDEO\EHPRUHFRPSOH[
5HVWDUWWKHÀOWHUHQJLQHWRDFWLYDWHWKHQHZFRQÀJXUDWLRQ ~/blacktest$ bin/mysqladmin -uroot -S black.sock shutdown ~/blacktest$ xterm -T BLACK -e bin/mysqld \ > --defaults-file=my.black \ > --console &
5HFRQQHFWWKHFOLHQWFRQQHFWHGWRWKHEODFNKROHHQJLQH7RGRWKLVMXVWLVVXHD SELECT1;FRPPDQG
([HFXWHWKHIROORZLQJFRPPDQGWRKRRNXSWKHÀOWHUWRWKHPDVWHU%HVXUHWRÀOOLQWKH YDOXHV\RXZURWHGRZQDPRPHQWDJRLQWKHVWDWHPHQW black> -> -> -> -> -> ->
CHANGE MASTER TO master_host='localhost', master_port=3307, master_user='repblack', master_password='blackpass', master_log_file='master-bin.000001', master_log_pos=1074;
5HSOLFDWLRQ 5HWULHYHLQIRUPDWLRQUHTXLUHGWRVHWXSWKHÀOWHUVODYHSRUWLRQ:ULWHGRZQWKHUHVXOWV RIWKHSHOWMASTERSTATUSFRPPDQGWKH\ZLOOEHQHHGHGODWHU black> FLUSH LOGS; black> SHOW MASTER STATUS; +------------------+----------+---+---+ | File | Position | … | … | +------------------+----------+---+---+ | black-bin.000003 | 98 | | | +------------------+----------+---+---+
6WDUWWKHVODYHWKUHDGRQWKHÀOWHUHQJLQHDQGYHULI\WKDWHYHU\WKLQJLVJRLQJZHOO black> START SLAVE; black> SHOW SLAVE STATUS \G ************************** 1. row *************************** Slave_IO_State: Waiting for master to send event Master_Host: localhost Master_User: repblack Master_Port: 3307 Connect_Retry: 60 Master_Log_File: master-bin.000001 Read_Master_Log_Pos: 1074 Relay_Log_File: black-relay.000003 Relay_Log_Pos: 236 Relay_Master_Log_File: master-bin.000001 Slave_IO_Running: Yes Slave_SQL_Running: Yes ... Replicate_Do_Table: repdb.tblB Replicate_Ignore_Table: repdb.tblA ... Last_Errno: 0 Last_Error: Skip_Counter: 0 Exec_Master_Log_Pos: 1074 Relay_Log_Space: 236 ... Seconds_Behind_Master: 0
$WWKLVSRLQWZHKDYHVXFFHVVIXOO\HVWDEOLVKHGDUHSOLFDWLRQFRQQHFWLRQEHWZHHQWKH PDVWHUGDWDEDVHDQGWKHEODFNKROHEDVHGÀOWHULQVWDQFH
&KDSWHU
&KHFNWKDWQRWKLQJKDV\HWEHHQZULWWHQWR WKHÀOWHU VELQORJV%HFDXVHZHLVVXHGDFLUSHLOGSFRPPDQGRQWKHÀOWHULQVWDQFH WKHUHVKRXOGEHQRWKLQJLQWKHPRVWUHFHQWELQORJÀOH9HULI\WKLVDVIROORZV ~/blacktest$ bin/mysqlbinlog black-bin.000003
7HVWWKHÀOWHUVHWXSZLWKVRPHVWDWHPHQWVLVVXHGRQWKHPDVWHU master> UPDATE repdb.tblA -> SET label='modified label 3' -> WHERE id=3; master> INSERT INTO repdb.tblB -> VALUES ('John', 39);
:HZRXOGH[SHFWWRVHHWKHINSERTLQWKHELQORJÀOHRIWKHÀOWHULQVWDQFHEXWQRWWKH UPDATEVWDWHPHQWEHFDXVHLWPRGLÀHVtblAZKLFKLVWREHLJQRUHG
9HULI\WKDWWKHUXOHVZRUNDVH[SHFWHGE\KDYLQJDQRWKHUORRNDWWKHÀOWHU VELQORJV ~/blacktest$ bin/mysqlbinlog black-bin.000003
5HSOLFDWLRQ 7KLVORRNVSUHFLVHO\DVH[SHFWHG³WKHINSERTLVSUHVHQWWKHUPDATELVQRZKHUHWR EHVHHQ
6HWXSWKHFRQÀJXUDWLRQRIDVODYHXVLQJWKHVHVHWWLQJV [client] port socket
= 3309 = /home/ds/blacktest/slave.sock
[mysqld_safe] socket
= /home/ds/blacktest/slave.sock
[mysqld] user pid-file socket port basedir datadir tmpdir language
= = = = = = = =
bind-address
= 127.0.0.1
server-id relay-log
= 3 = /home/ds/blacktest/slave-relay.log
mysql /home/ds/blacktest/slave.pid /home/ds/blacktest/slave.sock 3309 /home/ds/blacktest /home/ds/blacktest/data.slave /tmp /home/ds/blacktest/share/mysql/english
1RWLFHWKDWDOORFFXUUHQFHVRIPDVWHUKDYHEHHQUHSODFHG ZLWKVODYH
$JDLQWKHserver-idVHWWLQJKDVEHHQFKDQJHGDQGWKHlog-slave-updates skip-innodbDQGdefault-storage-engineRSWLRQVWKDWZHUHSDUWRIWKHÀOWHU LQVWDQFH VFRQÀJXUDWLRQDUHQRWLQFOXGHG$OVRWKHlog-binSDUDPHWHUKDVEHHQ UHPRYHGEHFDXVHFKDQJHVRQWKHVODYHQHHGQRWEHUHFRUGHGVHSDUDWHO\ 6WDUWXSWKHVODYHHQJLQH ' slave> CREATE DATABASE repdb; slave> USE repdb;
$WWKLVSRLQWWKHVODYHLVVHWXSDQGKDVDQHPSW\repdbGDWDEDVH )LOOXSWKHVODYHGDWDEDVHZLWKWKHLQLWLDOVQDSVKRWRIWKHPDVWHU:HQHHGWRORDGWZR ÀOHVKHUH7KHGHWDLOVRIZK\DUHH[SODLQHGIXUWKHUGRZQLQWKH+RZLWZRUNVVHFWLRQ slave> source master_struct.sql; ... slave> source master_data.sql; ...
9HULI\WKDW\RXFDQÀQGWKHGDWDIURPWKHPDVWHURQWKHVODYHQRZE\GRLQJDSELECT * FROMÀUVWWDEOHrepdb.tblADQGWKHQrepdb.tblB
7KHÀUVWSELECTVKRZVQRUHFRUGVEHFDXVHtblAZDVH[FOXGHGIURPWKHGXPS 7DEOHtblBFRQWDLQVWKHWKUHHUHFRUGVZHLQVHUWHGRQWKHPDVWHU
&UHDWHDUHSOLFDWLRQXVHUDFFRXQWRQWKHÀOWHULQVWDQFHIRUWKHVODYHVWRXVH black> -> -> ->
GRANT REPLICATION SLAVE ON *.* TO 'repslave'@'localhost' IDENTIFIED BY 'slavepass';
&RQQHFWWKHVODYHWRWKHÀOWHUHQJLQH%HVXUHWRLQVHUWWKHFRUUHFW YDOXHVIRUMASTER_LOG_FILEDQGMASTER_LOG_POSLQWKHVWDWHPHQW7KRVHDUH WKHYDOXHV\RXZURWHGRZQZKHQ\RXLVVXHGWKHSHOWMASTERSTATUSFRPPDQGRQ WKHÀOWHUVHUYHUEHIRUHVWDUWLQJWKHUHSOLFDWLRQWKHUH
5HSOLFDWLRQ slave> CHANGE MASTER TO -> master_host='localhost', -> master_port=3308, -> master_user='repslave', -> master_password='slavepass', -> master_log_file='black-bin.000003', -> master_log_pos=98; Query OK, 0 rows affected (0.01 sec)
6WDUWWKHVODYHDQGYHULI\WKDWLWVWDUWVXSFRUUHFWO\ slave> START SLAVE slave> SHOW SLAVE STATUS \G ************************** 1. row *************************** Slave_IO_State: Waiting for master to send event ... Relay_Master_Log_File: black-bin.000003 Slave_IO_Running: Yes Slave_SQL_Running: Yes ... Seconds_Behind_Master: 0
$VVRRQDVWKHSUHYLRXVVWHSLVFRPSOHWHWKHUHSOLFDWLRQVKRXOGDOUHDG\KDYHXSGDWHG tblBRQWKHVODYHDQGLQVHUWHGWKHQHZ -RKQ UHFRUG9HULI\LWOLNHWKLV slave> SELECT * FROM repdb.tblB;
$SSDUHQWO\WKHUHSOLFDWLRQZRUNV "D:\MySQLBackup\MySQLDumpAllDatabases.sql.gz"
+RZLWZRUNV 7KHRQO\GLIIHUHQFHLQWKHDERYHFRPPDQGOLQHLQFRPSDULVRQWRWKHSUHYLRXVUHFLSHV EHVLGH WKHVOLJKWO\FKDQJHGWDUJHWÀOHQDPHWRUHÁHFWWKHFRPSUHVVHGFRQWHQW OLHVZLWKLQ WKH| gzip --fastSRUWLRQ7KLVUHGLUHFWVWKHRXWSXWRIWKHmysqldumpFRPPDQGWRWKH J]LSSURJUDPZKLFKFRPSUHVVHVGDWDRQWKHÁ\7KHFRPSUHVVHGGDWDVWUHDPLVWKHQZULWWHQ WRWKHJLYHQÀOHUHVXOWLQJLQDVLJQLÀFDQWO\UHGXFHGVL]H
7KHUH VPRUH« :KLOHWKHUHFLSHLWVHOILVQRWWRRFRPSOLFDWHGWKHUHDUHVRPHDVSHFWVWRFRQVLGHUZKLFKDUH GLVFXVVHGLQWKHIROORZLQJVHFWLRQV
%DFNLQJ8SDQG5HVWRULQJ0\64/'DWD
$FKLHYLQJEHWWHUFRPSUHVVLRQUDWLR $VDEDOOSDUNÀJXUHIRUWKHFRPSUHVVLRQUDWLRIRUW\SLFDOGDWDEDVHVWKHFRPSUHVVHGGXPS ZLOOSUREDEO\EHUHGXFHGWRRQHWKLUGRILWVXQFRPSUHVVHGVL]H7RDFKLHYHEHWWHUFRPSUHVVLRQ DWWKHH[SHQVHRIUHGXFHGSHUIRUPDQFHDQGKLJKHU&38ORDG\RXFRXOGDOVROHDYHRXWWKH --fastRSWLRQRIJ]LSUHVXOWLQJLQDFRPSUHVVHGÀOHWKDWZLOOWDNHOHVVWKDQRIWKH RULJLQDOVL]H,IWKHVL]HRIWKHEDFNXSLVDQH[WUHPHO\FULWLFDOLVVXHIRU\RX\RXPLJKWHYHQ WU\WKH--bestRSWLRQDVDUHSODFHPHQWIRU--fast͕EXWEHZDUQHGWKDWWKLVPLJKWOHDGWRD GUDPDWLFLQFUHDVHLQH[HFXWLRQWLPHVZLWKPRVWO\PLQLPDOLPSURYHPHQWVLQVL]H WKLVLVZK\ZH UHFRPPHQGQRWXVLQJWKLVRSWLRQ
&RQVLGHULQJSHUIRUPDQFHIDFWRUV :LWKWKHDGGLWLRQDOFRPSUHVVLRQSDUWDGGHGWRWKHDERYHFRPPDQG\RXZRXOGLQWXLWLYHO\ H[SHFWWKDWWKLVQHFHVVDULO\KDVDQHJDWLYHLPSDFWRQWKHGXPSSHUIRUPDQFH6XUSULVLQJO\ WKLVLVQRWDOZD\VWKHFDVHDVWKHUHGXFHGVL]HRIWKHUHVXOWLQJÀOHDOVRGHFUHDVHVWKHDPRXQW RIGDWDZULWWHQWR\RXUGLVN7KLVUHGXFWLRQLQGLVN,2PLJKWPDNHXSIRUWKHDGGLWLRQDO FRPSUHVVLRQZRUNRUHYHQFDXVHDSHUIRUPDQFHLPSURYHPHQWmysql -u backup_usr -p"B4ckM3Up!" -e"START SLAVE SQL_THREAD"
+RZLWZRUNV 7KHUHFLSHEDVLFDOO\FRQVLVWVRIWKUHHSDUWV E\SHUIRUPLQJWKHRSHUDWLRQVGHVFULEHGLQVWHS UHSOLFDWLRQLVGLVDEOHG LQVWHSVDQGWKHEDFNXSLWVHOILVSHUIRUPHG LQVWHSUHSOLFDWLRQ LVHQDEOHGDJDLQ /HWXVKDYHDORRNDWWKHEDFNXSSDUWÀUVW,QVWHSWKHEDFNXSRIWKHGDWDVWRUHGLQWKH VODYHGDWDEDVHLVGRQHDVVKRZQLQWKHSUHYLRXVUHFLSHV7KHDGGLWLRQRIVWHSLVQHFHVVDU\ WRDOVREDFNXSWKHUHSOLFDWLRQVWDWHRIWKHVODYH,QGHWDLOWKLVFRSLHVWKHUHOD\ORJÀOHV [host]-relay-bin.indexDQG[hostname]-relay-bin.00x WKHrelay-log.info DQGWKHmaster.infoÀOHV7KHVHÀOHVDUHQHFHVVDU\WRUHFRYHUWKHVODYHIURPDFUDVK EHFDXVHZLWKRXWWKHVHÀOHVLWLVYHU\KDUGWRHVWDEOLVKDZRUNLQJUHSOLFDWLRQPHFKDQLVP ZLWKWKHPDVWHU,I\RXULQWHQWLRQLVWRSHUIRUPDEDFNXSRQO\IRUUHVWRULQJWKHPDVWHUWKHVH ÀOHVDUHQRWDEVROXWHO\QHFHVVDU\EXWZHVWURQJO\VXJJHVWLQFOXGLQJWKHVHÀOHVLQWKHEDFNXS DVZHOO
%DFNLQJ8SDQG5HVWRULQJ0\64/'DWD 7KDW VDERXWLWFRQFHUQLQJWKHEDFNXSLWVHOI1RZZKDWLVWKHPRWLYDWLRQIRUVWHSVDQG" 7KHGHDFWLYDWLRQRIWKHVODYHUHSOLFDWLRQLQVWHSLVQHFHVVDU\WRSUHYHQWLQFRQVLVWHQFLHV GXULQJWKHEDFNXS$VGLVFXVVHGEHIRUHERWKWKHFRS\DSSURDFKDQGWKHmysqldumpEDFNXS KDYHUHVWULFWLRQVFRQFHUQLQJGDWDFRQVLVWHQFLHVIRUQRQWUDQVDFWLRQDOWDEOHV OLNH0\,6$0 ZKHQFRQFXUUHQWXSGDWHVRFFXUZKLOHWKHGDWDLVUHDGsource
D:\MySQLBackup\MySQLDumpAllDatabases.sql
+RZLWZRUNV $JDLQWKLVLVDSUHWW\VWUDLJKWIRUZDUGUHFLSH FRQQHFWWRWKHGDWDEDVH VWHS DQGUHVWRUHWKH GDWDIURPWKHGXPS VWHS 7KHXVHRIWKHmysqlFRPPDQGOLQHFOLHQWLVQRWDEVROXWHO\QHFHVVDU\EXWSOHDVHQRWHWKDW \RXZLOOQRWEHDEOHWRLVVXHWKHsourceFRPPDQGIURPDQ\RWKHU64/FOLHQW7KLVLVEHFDXVH LWLVQRWDUHJXODU64/VWDWHPHQWEXWDNH\ZRUGUHFRJQL]HGE\WKHmysqlFRPPDQGOLQH FOLHQWLWVHOIgrep --invert-match "^INSERT INTO .op_detail. VALUES .*" D:\MySQLBackup\MySQLDump.sql | mysql -u admin4mysql -p"As,ysp4M"
7KLVFRPPDQGXVHVgrepWRÀOWHUDOOOLQHVLQFOXGLQJDQINSERT INTO 'op_detail'DW WKHVWDUWRIWKHOLQHOHDYLQJDOORWKHUOLQHVXQWRXFKHG$OORWKHUOLQHVDUHSLSHGWRWKHmysql FRPPDQGOLQHFOLHQWWKDWLPSRUWVWKHUHPDLQLQJGXPS͘ 3OHDVHQRWHWKDWLIGDWDKDVUHIHUHQFHVWRHQWULHVLQWKHH[FOXGHGWDEOHWKHLPSRUWRIWKH GXPSZLOOQRWIDLO,QWKLVFDVH\RXZRXOGHQGXSZLWKGDWDLQFRQVLVWHQFLHVWKDWPLJKWFDXVH HUURUVPXFKODWHUVRPDNHVXUHWRH[FOXGHRQO\WDEOHVWKDWDUHQRWUHIHUHQFHGE\IRUHLJQNH\V 2WKHUZLVH\RXKDYHWRVHDUFKIRUSRVVLEOHIRUHLJQNH\FRQVWUDLQWYLRODWLRQVPDQXDOO\ 6RPH LGHDVRQKRZWRDFKLHYHWKLVDUHSUHVHQWHGIRUH[DPSOHDWhttp://dev.mysql.com/doc/ refman/5.1/en/innodb-foreign-key-constraints.html
6HHDOVR f
'HÀQLQJDVSHFLÀFXVHUIRUDGPLQLVWUDWLYHWDVNV &UHDWLQJD64/GXPSRIDOOGDWDEDVHV
f
&UHDWLQJD64/GXPSRIVSHFLÀFGDWDEDVHV
f
&KDSWHU
3HUIRUPLQJDSRLQWLQWLPHUHFRYHU\XVLQJ WKHELQDU\ORJV 7KHSUHYLRXVUHFLSHVGHDOWZLWKKRZWRUHFRYHUGDWDIURPDEDFNXS$IWHUUHFRYHU\WKHGDWD VKRXOGEHLQWKHVDPHFRQGLWLRQDVLWZDVZKHQWKHEDFNXSZDVFUHDWHG8QIRUWXQDWHO\WKH GDWDWKDWZDVFKDQJHGDIWHUEDFNXSFUHDWLRQLVQRWUHVWRUHG 7RUHVWRUHWKHGDWDWRWKHSRLQWLQWLPHEHIRUHWKHUHFRYHU\\RXFDQXVHWKHbinlogÀOHVWKDW ZHUHFUHDWHGEHWZHHQEDFNXSDQGUHFRYHU\$VPHQWLRQHGLQWKH5RWDWLQJDQGSXUJHELQDU\ ORJVUHFLSHZHVXJJHVWHQDEOLQJELQDU\ORJJLQJHYHQLI\RXGRQRWKDYHDUHSOLFDWLRQVFHQDULR DVWKLVJLYHV\RXH[WHQGHGRSWLRQVLQEDFNXS ,QWKLVUHFLSHZHZLOOGLVFXVVKRZWRXVHWKHELQDU\ORJVWRUHVWRUHGDWDXSWRWKHODWHVW SRVVLEOHSRLQWLQWLPHEHIRUHWKHGDWDZDVORVW
*HWWLQJUHDG\ 7REHDEOHWRUHVWRUHWKHUHPDLQLQJGDWDXVLQJWKHELQDU\ORJJLQJLQIRUPDWLRQZHGHÀQLWHO\ QHHGWKHELQDU\ORJÀOHV,IDOOGDWDIURP\RXUVHUYHULVORVW IRUH[DPSOHLQFDVHRIDÀUH WKH binlogsDUHKRSHIXOO\DYDLODEOHIURPDWDSHRUDQ\RWKHUPHGLDWKDWLVQRWDIIHFWHGE\WKH GDWDORVV,IRQO\\RXUGDWDEDVHGDWDLVFRUUXSW\RXRIWHQKDYHWKHELQDU\ORJVVWLOOSUHVHQWRQ \RXUVHUYHU VGLVN ,IWKHELQDU\ORJVDUHVWLOOSUHVHQW\RXVKRXOGFRS\WKHELQDU\ORJÀOHVWRDGLIIHUHQWSRVLWLRQ EHIRUH\RXVWDUWUHFRYHULQJIURPDGXPS7KLVZD\\RXKDYHDOOELQDU\ORJÀOHVDYDLODEOHLQ H[DFWO\WKHVWDWHIURPWKHSRLQWLQWLPHEHIRUHWKHUHFRYHU\ :HDVVXPHWKHELQDU\ORJV HLWKHUDVDFRS\RIWKHbinlogÀOHVIURP\RXUVHUYHUGLVNRUDV UHFRYHUHGIURPEDFNXSPHGLD WREHVWRUHGXQGHUC:\tmp\binlogs\ 7RUHDGWKHFKDQJHVVWRUHGLQWKHELQDU\ORJVLQWRWKHGDWDEDVH\RXQHHGDQDSSURSULDWH 0\64/XVHUDJDLQ KHUH admin4mysqlZLWKSDVVZRUGAs,ysp4M $QGPRVWLPSRUWDQWO\\RXQHHGWKHSRVLWLRQDWZKLFKUHFRYHU\IURPWKHELQDU\ORJÀOHV VKRXOGVWDUW7KHGXPS\RXUGDWDEDVHZDVUHVWRUHGIURPVKRXOGKDYHEHHQFUHDWHGXVLQJ WKH--master-dataRSWLRQ VHHWKH7KHUH VPRUH«VHFWLRQRIWKH&UHDWLQJDQ64/GXPSRI DOOGDWDEDVHVUHFLSH RWKHUZLVHWKLVLQIRUPDWLRQLVQRWHDVLO\DYDLODEOH EXECUTE statement;
7KHÀUVWSETVWDWHPHQWFRQVWUXFWVDVWULQJZKLFKFRQWDLQVDSELECTVWDWHPHQW:KLOHLWLV QRWDOORZHGWRXVHYDULDEOHVIRUVWDWHPHQWVGLUHFWO\\RXFDQFRQVWUXFWDVWULQJWKDWFRQWDLQV DVWDWHPHQWDQGXVHYDULDEOHVIRUWKLV:LWKWKHQH[WWZROLQHV\RXSUHSDUHDVWDWHPHQWIURP WKHVWULQJDQGH[HFXWHLW
&KDSWHU
+DQGOLQJ18//YDOXHV :LWKRXWIXUWKHUKDQGOLQJNULLYDOXHVLQWKHGDWD\RXH[SRUWXVLQJWKHSUHYLRXVVWDWHPHQW ZRXOGVKRZXSDV"NLQWKHUHVXOWLQJÀOH7KLVFRPELQDWLRQLVQRWUHFRJQL]HGIRUH[DPSOHE\ 0LFURVRIW([FHOZKLFKEUHDNVWKHÀOH IRUW\SLFDOXVDJH 7RSUHYHQWWKLV\RXQHHGWRUHSODFH NULLHQWULHVE\DSSURSULDWHYDOXHV$VVXPLQJWKDWWKHWDEOHsample.table1FRQVLVWVRID QXPHULFFROXPQaDQGDFKDUDFWHUFROXPQb\RXVKRXOGXVHWKHIROORZLQJVWDWHPHQW mysql> SELECT IFNULL(a, 0), IFNULL(b, "NULL") FROM sample.table1 INTO OUTFILE 'C:/target.csv' FIELDS ENCLOSED BY '"' TERMINATED BY ';' ESCAPED BY '"' LINES TERMINATED BY '\r\n';
7KHGRZQVLGHWRWKLVDSSURDFKLVWKDW\RXKDYHWROLVWDOOÀHOGVLQZKLFKDNULLYDOXH PLJKWRFFXU
+DQGOLQJOLQHEUHDNV ,I\RXWU\WRH[SRUWYDOXHVWKDWFRQWDLQWKHVDPHFKDUDFWHUFRPELQDWLRQXVHGIRUOLQH WHUPLQDWLRQLQWKHSELECT … INTO OUTFILEVWDWHPHQW0\64/ZLOOWU\WRHVFDSHWKH FKDUDFWHUFRPELQDWLRQZLWKWKHFKDUDFWHUVGHÀQHGE\WKHESCAPED BYFODXVH+RZHYHU WKLVZLOOQRWDOZD\VZRUNWKHZD\LWLVLQWHQGHG
UNION SELECT @timestamp
">
UNION SELECT @rows
">
UNION SELECT @header
"> ">
UNION SELECT CONCAT('#', @counter := @counter + 1,
0DQDJLQJ'DWD ">
': ',
">
CONCAT_WS(' | ', c1, c2, c3),
">
' $')
">
FROM sample.table2
">
INTO OUTFILE '", file, "';");
->
SET @counter := 0;
->
PREPARE statement FROM @command;
->
EXECUTE statement;
->
END //
Query OK, 0 rows affected (0.00 sec)
mysql> delimiter ;
7KLVURXWLQHDOORZVSDVVLQJWKHÀOHQDPHZKHQFDOOLQJLW mysql> call sample.export_table2_FileAsParam("C:/data.out"); Query OK, 0 rows affected (0.00 sec)
,PSRUWLQJGDWDIURPDVLPSOH&69ÀOH $FRPPRQWDVNZKHQZRUNLQJZLWKGDWDEDVHVLVWRLPSRUWGDWDIURPGLIIHUHQWVRXUFHV 8QIRUWXQDWHO\WKLVGDWDZLOOW\SLFDOO\QRWEHSURYLGHGDVDFRQYHQLHQWVHWRIZHOOIRUPHG64/ VWDWHPHQWVWKDW\RXFDQVLPSO\UXQDJDLQVW\RXUGDWDEDVH7KHUHIRUHKHUH\RXZLOOKDYHWR GHDOZLWKGDWDLQDGLIIHUHQWIRUPDW $VDFRPPRQGHQRPLQDWRUFKDUDFWHUVHSDUDWHGYDOXHV &69 DUHVWLOODSUHYDOHQWZD\RI H[FKDQJLQJGDWD,QWKLVFKDSWHUZHZLOOVKRZ\RXKRZWRLPSRUWGDWDVWRUHGLQ&69ÀOHV$VD W\SLFDOH[DPSOHZHZLOOXVHWKHÀOHIRUPDW0LFURVRIW([FHOSURGXFHVZKHQVWRULQJÀOHVXVLQJ WKH*.CSVÀOHW\SH 7KLVUHFLSHLVWKHFRXQWHUSDUWRIWKH([SRUWLQJGDWDWRDVLPSOH&69ÀOHUHFLSHLQWKLVFKDSWHU
*HWWLQJUHDG\ 7RVWHSWKURXJKWKLVUHFLSHZHZLOOGHÀQLWHO\QHHGDÀOHWRLPSRUW KHUH C:/source.csv DQGDWDEOHWRLPSRUWWKHGDWDLQWR KHUH sample.table1 7KHVRXUFHÀOHDQGWDUJHWWDEOH KDYHWRKDYHDPDWFKLQJIRUPDWFRQFHUQLQJWKHQXPEHURIFROXPQVDQGWKHW\SHRIGDWD VWRUHGLQWKHP)XUWKHUPRUHDQDFFRXQWZLWKINSERTDQGFILESULYLOHJHVLVUHTXLUHG ZHZLOO DVVXPHDQDFFRXQWsample_installLQWKLVUHFLSH
&KDSWHU
7KHVRXUFHÀOHKDVWREHORFDWHGRQWKHPDFKLQHWKDWUXQV\RXU0\64/ VHUYHUQRWRQWKHFOLHQWVLGH
+RZWRGRLW &RQQHFWWRWKHGDWDEDVHXVLQJWKHsample_installDFFRXQW ,VVXHWKHIROORZLQJ64/FRPPDQG mysql> LOAD DATA INFILE 'C:/source.csv' INTO TABLE sample.table1 FIELDS ENCLOSED BY '"' TERMINATED BY ';' ESCAPED BY '"' LINES TERMINATED BY '\r\n'; Query OK, 20 rows affected (0.06 sec) Records: 20
Deleted: 0
Skipped: 0
Warnings: 0
+RZLWZRUNV 7KHLOAD DATA INFILEFRPPDQGZRUNVDQDORJRXVWRWKHSELECT … INTO OUTFILE FRPPDQGGLVFXVVHGLQWKHSUHYLRXVUHFLSHVEXWDVDPHDQVIRULPSRUWLQJGDWDUDWKHUWKDQ H[SRUWLQJ7KHIRUPDWRSWLRQVDYDLODEOHIRUERWKFRPPDQGVDUHLGHQWLFDOVR\RXFDQW\SLFDOO\ LPSRUWGDWDH[SRUWHGE\DSELECT … INTO OUTFILEVWDWHPHQWXVLQJDLOAD DATA INFILEFRPPDQGZLWKWKHVDPHIRUPDWRSWLRQV $VPRVWÀOHVFRQVLVWRIOLQHVWHUPLQDWHGE\DVHTXHQFHRIDFDUULDJHUHWXUQDQGDOLQHIHHG FKDUDFWHUZHXVHWKHLINES TERMINATED BY '\r\n'RSWLRQ7KHFKRLFHRIWKHVHPLFRORQ FKDUDFWHU³DVDVHSDUDWRUIRUGLIIHUHQWÀHOGVRIHYHU\OLQH TERMINATED BY ';' ³LVPDLQO\ GXHWRWKHIDFWWKDW([FHOXVHVWKLVIRUPDW,I\RXKDSSHQWRUHFHLYH&69ÀOHVWKDWIRUH[DPSOH XVHDFRPPDLQVWHDG\RXKDYHWRDGMXVWWKLVDFFRUGLQJO\ 7KHWHUPFIELDS ENCLOSED BY '"'WHOOVWKHLPSRUWWRORRNIRUGRXEOHTXRWHVDWWKH VWDUWRIHYHU\ÀHOGLPSRUWHG,IWKHUHLVRQHWKHÀHOGLVFRQVLGHUHGWRHQGDWWKHQH[WGRXEOH TXRWH7REHDEOHWRKDYHGRXEOHTXRWHVLQVLGHDÀHOGYDOXHZHGHÀQHDQHVFDSHFKDUDFWHU ESCAPED BY '"' :LWKWKLVFRQVWHOODWLRQDVHTXHQFHRIWZRGRXEOHTXRWHVLVQRWWUHDWHG DVWKHHQGRIWKHÀHOGEXWDVDGRXEOHTXRWHFKDUDFWHUDVSDUWRIWKHYDOXH
7KHUH VPRUH 7KHGDWDLVUHDGIURPWKHÀOHXVLQJWKHGHIDXOWFKDUDFWHUVHWRIWKHGDWDEDVH,IWKHÀOHXVHVD GLIIHUHQWFKDUDFWHUHQFRGLQJ\RXFDQVSHFLI\WKLVE\DGGLQJDCHARACTER SETFODXVHDIWHU WKHWDEOHGHÀQLWLRQ LOAD DATA INFILE … INTO TABLE sample.table1 CHARACTER SET utf8; 3OHDVHQRWHWKDWWKHFKDUDFWHUVHWVucs2utf16DQGutf32DUHQRWVXSSRUWHG DVRI0\64/YHUVLRQ
0DQDJLQJ'DWD
6HHDOVR f
([SRUWLQJGDWDWRDVLPSOH&69ÀOH
,PSRUWLQJGDWDIURPFXVWRPÀOHIRUPDWV ,QWKHSUHYLRXVUHFLSH,PSRUWLQJGDWDIURPDVLPSOH&69ÀOHZHGLVFXVVHGDZD\RILPSRUWLQJ GDWDIURPDQLFHO\IRUPDWWHGÀOH8QIRUWXQDWHO\\RXVRPHWLPHVKDYHWRGHDOZLWKIDUOHVV FRQYHQLHQWGDWDVRXUFHV,QWKLVUHFLSHZHZLOOSUHVHQWVRPHPRUHDGYDQFHGWRSLFVRI LPSRUWLQJGDWDIURPÀOHVZLWKDOHVVVWULFWVWUXFWXUH 2EYLRXVO\LWLVQRWSRVVLEOHWRSUHVHQWDXQLYHUVDOUHFLSHIRUHYHU\ÀOHIRUPDWLPDJLQDEOHVR ZHZLOOXVHDQH[DPSOHWKDWFRYHUVVRPHRIWKHFRPPRQSUREOHPVRQHKDVWRWDFNOHZKHQ LPSRUWLQJGDWDIURPFXVWRPÀOHV)RUWKLVZHZLOOUHIHUWRWKHVDPHK\SRWKHWLFDOIRUPDWDVLQ ([SRUWGDWDWRDFXVWRPÀOHIRUPDWZKLFKGHÀQHVIRXULQLWLDOOLQHV FRQWDLQLQJQDPHRIWKH ÀOHDWLPHVWDPSDGHVFULSWLRQDQGWKHQXPEHURIURZV DKHDGHUOLQHZLWKWKHQDPHRIWKH FROXPQVDQGVXEVHTXHQWO\WKHURZVZLWKWKHDFWXDOGDWDWRLPSRUW(DFKGDWDURZVWDUWVZLWK DKDVKFKDUDFWHU # WKHOLQHQXPEHUDFRORQDQGDVSDFH7KHGDWDYDOXHVWKDWIROORZWKH URZQXPEHUDUHVHSDUDWHGE\DSLSH | FKDUDFWHUDQGWKHURZFORVHVZLWKDGROODUVLJQ $
*HWWLQJUHDG\ $JDLQWKHDFFRXQWXVHGLQWKHUHFLSHQHHGVWKHFILESULYLOHJH EHVLGHVWKHINSERT SHUPLVVLRQIRUWKHWDEOHWKHGDWDVKRXOGEHLPSRUWHGLQWR :LWKD64/FOLHQWDÀOHZLWK WKHDSSURSULDWHIRUPDWDQGDWDEOHDVWKHLPSRUWWDUJHWZHDUHUHDG\WRJR$VLQSUHYLRXV UHFLSHVZHXVHsample_installDVWKHDFFRXQWQDPHC:/source.txtDVWKHVRXUFHÀOH DQGsample.table2 FRQVLVWLQJRIWKUHHFROXPQVc1c2DQGc3 DVWKHWDUJHWWDEOH:H DVVXPHWKHVRXUFHÀOHWRKDYHWKHIROORZLQJFRQWHQW Filename: C:/source.txt Description: This is a file for test import to sample.table2, columns c1, c2, and c3 2009-06-14 13:25:05 Row count: 3 #Row Nr: Column c1 | Column c2 | Column c3 $ #1: 209 | Some text in my test data | Some more text $ #2: 308 | Next test text for testing | Text to test $ #3: 406 | "A water | pipe" | Really? $
+RZWRGRLW &RQQHFW\RXUIDYRULWHFOLHQW IRUH[DPSOHWKHmysqlFRPPDQGOLQHFOLHQW WR\RXU 0\64/VHUYHUXVLQJWKHsample_installDFFRXQW
&KDSWHU
([HFXWHWKHIROORZLQJ64/FRPPDQG mysql> LOAD DATA INFILE "C:/source.txt" ->
INTO TABLE sample.table2
->
FIELDS TERMINATED BY ' | '
->
OPTIONALLY ENCLOSED BY '"'
->
LINES STARTING BY ':'
->
TERMINATED BY '\r\n'
->
IGNORE 5 LINES
->
SET c3=TRIM(TRAILING ' $' FROM c3);
Query OK, 3 rows affected (0.05 sec) Records: 3
Deleted: 0
Skipped: 0
Warnings: 0
+RZLWZRUNV /HWXVGLVVHFWWKHDERYHVWDWHPHQWE\KDYLQJDORRNDWWKHVRXUFHÀOH ÀUVWRIDOOZHZDQW WRLPSRUWGDWDIURPWKHÀOHC:\source.txtLQWRWKHWDEOHsample.table2ZKLFKLV UHSUHVHQWHGE\WKHÀUVWWZROLQHV LOAD DATA INFILE … INTO TABLE … $WWKHWRSRIWKHÀOHZHKDYHÀYHOLQHV WKHLQLWLDOIRXUOLQHVZLWKLQIRUPDWLRQDERXWWKHÀOHSOXV WKHKHDGHU WKDWVKRXOGQRWEHLPSRUWHGLQWRWKHWDUJHWWDEOH7RDFKLHYHWKLVWKHIGNORE 5 linesRSWLRQLVDGGHG 7KHUHPDLQLQJOLQHVDUHSUHÀ[HGZLWKDKDVKFKDUDFWHUWKHURZQXPEHUDQGDFRORQ7KLVSDUW RIHYHU\OLQHKDVWREHLJQRUHGZKLFKLVZKDWWKHLINES STARTING BY ':'RSWLRQGRHV LW WHOOV0\64/WRLJQRUHWKHÀUVWFRORQRIWKHOLQHDQGDQ\FKDUDFWHUEHIRUHLW%\GRLQJWKLVWKH URZQXPEHUSUHÀ[LVVNLSSHG $IWHUWKHSUHÀ[WKHOLQHVFRQWDLQWKHDFWXDOYDOXHVVHSDUDWHGE\SLSHFKDUDFWHUV7KHFIELDS TERMINATED BY ' | 'RSWLRQWHOOV0\64/KRZWRLGHQWLI\DÀHOGVHSDUDWRU:LWKWKH DGGLWLRQDOVHWWLQJOPTIONALLY ENCLOSED BY '"'WKHYDOXHLWVHOIPLJKWFRQWDLQWKLVÀHOG VHSDUDWRUVHTXHQFH³LIWKHZKROHYDOXHLVHQFORVHGE\GRXEOHTXRWHV WKLVLVWKHFDVHLQWKH ODVWURZRIWKHVDPSOHÀOH $WWKLVSRLQWWKHUHLVRQO\RQHSUREOHPOHIW WKHOLQHVHQGZLWKDGROODUVLJQZKLFKLVQRW SDUWRIWKHODVWYDOXH$QLQWXLWLYHDSSURDFKZRXOGEHWRLQFOXGHWKLVFKDUDFWHULQWKHOLQH WHUPLQDWLRQVHTXHQFHZKLFKPHDQVWRXVH$\r\nDVDOLQHHQGLQJ LQVWHDGRI\r\n 8QIRUWXQDWHO\WKLVGHÀQLWLRQRIDOLQHHQGGRHVQRWZRUNDVH[SHFWHGIRURXUH[DPSOHDVLW ZRXOGEUHDNWKHLQWHUSUHWDWLRQRIWKHÀUVWÀYHOLQHVZKLFKDUHQRWWHUPLQDWHGWKHVDPHZD\ $VDUHVXOWWKHÀUVWVL[OLQHVZRXOGEHFRQVLGHUHGDVRQHVLQJOHOLQHE\WKHLPSRUWEHFDXVH RQO\WKHVL[WKOLQHDFWXDOO\HQGVZLWKDFKDUDFWHUVHTXHQFHRI$\r\n7REHDEOHWRH[SOLFLWO\ H[FOXGHWKHKHDGHUOLQHVIURPWKHLPSRUWZHKDYHWRUHO\RQWKHWUDGLWLRQDOOLQHHQGLQJ GHÀQHGE\WKH[LINES] TERMINATED BY '\r\n'RSWLRQ
0DQDJLQJ'DWD +HQFHWKHRSWLRQVIRUGHÀQLQJWKHÀHOGVHSDUDWRUVDQGWKHEHJLQQLQJDQGWHUPLQDWLRQRI DOLQHGRQRWDOORZXVWRJHWULGRIWKHFORVLQJGROODUVLJQ7KXVLWLVFRQVLGHUHGSDUWRIWKH ODVWYDOXHZKLFKLVDVVLJQHGWRFROXPQc37RÀQDOO\JHWULGRIWKLVSRVWÀ[WKHSETFODXVH RIWKHLOAD DATA INFILE FRPPDQGFRPHVLQKDQG\ZKLFKDOORZVWRFOHDUO\GHÀQH WKHYDOXHVWKDWDUHDVVLJQHGWRWKHFROXPQVLQWKHWDUJHWWDEOH7KHFORVLQJRSWLRQSET c3=TRIM(TRAILING ' $' FROM c3);GHÀQHVDZD\WRVWULSWKHXQZDQWHGSRVWÀ[IURP WKHODVWÀHOG ,IZHSXWLWDOOWRJHWKHUWKHLPSRUWZRUNVDVLQWHQGHG
7KHUH VPRUH $VZLWKH[SRUWLQJGDWDLWLVUHFRPPHQGHGWRFRQVLGHUXVLQJDQH[WHUQDOSURJUDPPLQJ ODQJXDJHWRLPSRUWPRUHFRPSOH[GDWDVWUXFWXUHVLQWR0\64/:KLOHLWLVSRVVLEOHWRLPSRUW UDWKHUVRSKLVWLFDWHGÀOHIRUPDWVXVLQJ0\64/FRPPDQGVDVZHOOLWLVRIWHQIDUPRUHHIÀFLHQW WRKDYHDIXOOEORZQSURJUDPPLQJODQJXDJHDWKDQGWRVROYHWKHWDVNRISDUVLQJLQSXWÀOHV 7KLVLVPRVWQRWDEO\WKHFDVHZKHQLWFRPHVWR;0/ÀOHV
)RULPSRUWLQJGDWDIURP;0/ÀOHVFRQVLGHUXVLQJDQ;6/7SURFHVVRU WRSURGXFHFRUUHVSRQGLQJ64/FRPPDQGV
6HHDOVR f
,PSRUWLQJGDWDIURPDVLPSOH&69ÀOH
f
([SRUWLQJGDWDWRDFXVWRPÀOHIRUPDW
&KDSWHU
,QVHUWLQJQHZGDWDDQGXSGDWLQJGDWDLILW DOUHDG\H[LVWV 0DQLSXODWLQJGDWDLQDGDWDEDVHLVSDUWRIHYHU\GD\ZRUNDQGWKHEDVLF64/PHDQVRI INSERTUPDATEDQGDELETEPDNHWKLVDSUHWW\VWUDLJKWIRUZDUGDOPRVWWULYLDOWDVN³EXW LVWKLVDOZD\VWUXH" :KHQFRQVLGHULQJGDWDPDQLSXODWLRQPRVWRIWKHWLPHZHWKLQNRIDVLWXDWLRQZKHUHZH NQRZWKHFRQWHQWRIWKHGDWDEDVH:LWKWKLVLQIRUPDWLRQLWLVXVXDOO\SUHWW\HDV\WRÀQG DZD\RIFKDQJLQJWKHGDWDWKHZD\\RXLQWHQGWR%XWZKDWLI\RXKDYHWRFKDQJHGDWDLQ FLUFXPVWDQFHVZKHUH\RXGRQRWNQRZWKHDFWXDOGDWDEDVHFRQWHQWEHIRUHKDQG" INSERT INTO sample.config VALUES ("maxPriceDiscount", "25%") ON DUPLICATE KEY UPDATE value='25%'; Query OK, 1 row affected (0.05 sec)
+RZLWZRUNV 7KLVFRPPDQGLVHDVLO\H[SODLQHGEHFDXVHLWVLPSO\GRHVZKDWLWVD\V LWLQVHUWVDQHZURZLQ WKHWDEOHXVLQJWKHJLYHQYDOXHVDVORQJDVWKLVGRHVQRWFDXVHDGXSOLFDWHHQWU\LQHLWKHUWKH SULPDU\NH\RUDQRWKHUXQLTXHLQGH[,IDGXSOLFDWHUHFRUGH[LVWVWKHH[LVWLQJURZLVXSGDWHG DFFRUGLQJWRWKHFODXVHVGHÀQHGDIWHUON DUPLICATE KEY UPDATE :KLOHLWLVVRPHWLPHVWHGLRXVWRHQWHUVRPHRIWKHGDWDDQGFROXPQVWZRWLPHV RQFHIRUWKH INSERTDQGDVHFRQGWLPHIRUWKHUPDATE WKLVVWDWHPHQWDOORZVIRUDORWRIÁH[LELOLW\ZKHQLW FRPHVWRWKHPDQLSXODWLRQRISRWHQWLDOO\H[LVWLQJGDWD 3OHDVHQRWHWKDWZKHQH[HFXWLQJWKHDERYHVWDWHPHQWWKHUHVXOWGLIIHUVVOLJKWO\ZLWKUHVSHFW WRWKHQXPEHURIDIIHFWHGURZVGHSHQGLQJRQWKHDFWXDOGDWDSUHVHQWLQWKHGDWDEDVH :KHQ WKHUHFRUGGRHVQRWH[LVW\HWLWLVLQVHUWHGZKLFKUHVXOWVLQRQHDIIHFWHGURZ%XWLIWKHUHFRUG LVXSGDWHGUDWKHUWKDQLQVHUWHGLWUHSRUWVWZRDIIHFWHGURZVLQVWHDGHYHQLIRQO\RQHURZ JHWVXSGDWHG
7KHUH VPRUH 7KHINSERT INTO … ON DUPLICATE UPDATEFRQVWUXFWGRHVQRWZRUNZKHQWKHUHLV QRUNIQUERUPRIMARY KEYGHÀQHGRQWKHWDUJHWWDEOH,I\RXKDYHWRSURYLGHWKHVDPH VHPDQWLFVZLWKRXWKDYLQJDSSURSULDWHNH\GHÀQLWLRQVLQSODFHLWLVUHFRPPHQGHGWRXVHWKH WHFKQLTXHVGLVFXVVHGLQWKHQH[WUHFLSH
6HHDOVR f
,QVHUWLQJGDWDEDVHGRQH[LVWLQJGDWDEDVHFRQWHQW
&KDSWHU
,QVHUWLQJGDWDEDVHGRQH[LVWLQJGDWDEDVH FRQWHQW ,QWKHSUHYLRXVUHFLSH,QVHUWLQJQHZGDWDDQGXSGDWLQJGDWDLILWDOUHDG\H[LVWVZHGLVFXVVHG DPHWKRGWRHLWKHULQVHUWRUXSGDWHUHFRUGVGHSHQGLQJRQZKHWKHUWKHUHFRUGVDOUHDG\H[LVWLQ WKHGDWDEDVH$VLPLODUSUREOHPDULVHVZKHQ\RXQHHGWRLQVHUWGDWDWR\RXUGDWDEDVHEXWWKH GDWDWRLQVHUWGHSHQGVRQWKHGDWDLQ\RXUGDWDEDVH $VDQH[DPSOHFRQVLGHUDVLWXDWLRQLQZKLFK\RXQHHGWRLQVHUWDUHFRUGZLWKDFHUWDLQ PHVVDJHLQWRDWDEOHlogMsgsEXWWKHPHVVDJHLWVHOIVKRXOGEHGLIIHUHQWGHSHQGLQJRQWKH FXUUHQWV\VWHPODQJXDJHWKDWLVVWRUHGLQDFRQÀJXUDWLRQWDEOH config ,WLVIDLUO\HDV\WRDFKLHYHDVLPLODUEHKDYLRUIRUDQUPDATEVWDWHPHQWEHFDXVHWKLVVXSSRUWVD WHEREFODXVHWKDWFDQEHXVHGWRRQO\SHUIRUPDQXSGDWHLIDFHUWDLQSUHFRQGLWLRQLVPHW UPDATE logMsgs SET message= CONCAT('Last update: ', NOW()) WHERE EXISTS (SELECT value FROM config WHERE name='lang' AND value = 'en'); UPDATE logMsgs SET message= CONCAT('Letztes Update: ', NOW()) WHERE EXISTS (SELECT value FROM config WHERE name='lang' AND value = 'de'); UPDATE logMsgs SET message= CONCAT('Actualisation derniere: ', NOW()) WHERE EXISTS (SELECT value FROM config WHERE name='lang' AND value = 'fr');
8QIRUWXQDWHO\WKLVDSSURDFKLVQRWDSSOLFDEOHWRINSERTFRPPDQGVDVWKHVHGRQRWVXSSRUW DWHEREFODXVH'HVSLWHWKLVPLVVLQJRSWLRQWKHIROORZLQJUHFLSHGHVFULEHVDPHWKRGWRPDNH INSERTVWDWHPHQWVH[HFXWHRQO\LIDQDSSURSULDWHSUHFRQGLWLRQLQWKHGDWDEDVHLVPHW
*HWWLQJUHDG\ $VEHIRUHZHDVVXPHDGDWDEDVHD64/FOLHQW mysql DQGD0\64/XVHUZLWKVXIÀFLHQW SULYLOHJHV INSERTDQGSELECTLQWKLVFDVH $GGLWLRQDOO\ZHQHHGDWDEOHWRLQVHUWGDWD LQWR KHUH logMsgs DQGDFRQÀJXUDWLRQWDEOHconfig SOHDVHUHIHUWRWKHSUHYLRXVUHFLSH IRUGHWDLOV
0DQDJLQJ'DWD
+RZWRGRLW &RQQHFWWR\RXUGDWDEDVHXVLQJ\RXU64/FOLHQW ([HFXWHWKHIROORZLQJ64/FRPPDQGV mysql> INSERT INTO sample.logMsgs(message) ->
SELECT CONCAT('Last update: ', NOW())
->
FROM sample.config WHERE name='lang' AND value='en';
Query OK, 0 rows affected (0.00 sec) Records: 0
Duplicates: 0
Warnings: 0
mysql> INSERT INTO sample.logMsgs(message) ->
SELECT CONCAT('Letztes Update: ', NOW())
->
FROM sample.config WHERE name='lang' AND value='de';
Query OK, 1 row affected (0.05 sec) Records: 1
Duplicates: 0
Warnings: 0
mysql> INSERT INTO sample.logMsgs(message) ->
SELECT CONCAT('Dernière actualisation: ', NOW())
->
FROM sample.config WHERE name='lang' AND value='fr';
Query OK, 0 rows affected (0.00 sec) Records: 0
Duplicates: 0
Warnings: 0
+RZLWZRUNV 2XUJRDOLVWRKDYHDQINSERTVWDWHPHQWWDNHLQWRDFFRXQWWKHSUHVHQWODQJXDJHVWRUHGLQ WKHGDWDEDVH7KHWULFNWRGRVRLVWRXVHDSELECTVWDWHPHQWDVLQSXWIRUWKHINSERT7KH SELECTFRPPDQGSURYLGHVDWHEREFODXVHVR\RXFDQXVHDFRQGLWLRQWKDWRQO\PDWFKHV IRUWKHUHVSHFWLYHODQJXDJH2QHUHVWULFWLRQRIWKLVVROXWLRQLVWKDW\RXFDQRQO\LQVHUWRQH UHFRUGDWDWLPHVRWKHVL]HRIVFULSWVPLJKWJURZFRQVLGHUDEO\LI\RXKDYHWRLQVHUWORWVRI GDWDDQGRUKDYHWRFRYHUPDQ\DOWHUQDWLYHV
7KHUH VPRUH ,I\RXKDYHPRUHWKDQMXVWDIHZYDOXHVWRLQVHUWLWLVPRUHFRQYHQLHQWWRKDYHWKHGDWDLQRQH SODFHUDWKHUWKDQGLVWULEXWHGRYHUVHYHUDOLQGLYLGXDOINSERTVWDWHPHQWV,QWKLVFDVHLWPLJKW PDNHVHQVHWRFRQVROLGDWHWKHGDWDE\SXWWLQJLWLQVLGHDWHPSRUDU\WDEOH WKHÀQDOINSERT VWDWHPHQWXVHVWKLVWHPSRUDU\WDEOHWRVHOHFWWKHDSSURSULDWHGDWDURZVIRULQVHUWLRQLQWRWKH WDUJHWWDEOH7KHGRZQVLGHRIWKLVDSSURDFKLVWKDWWKHXVHUQHHGVWKHCREATE TEMPORARY TABLESSULYLOHJHEXWLWW\SLFDOO\FRPSHQVDWHVZLWKPXFKFOHDQHUVFULSWV
&KDSWHU
$IWHUFUHDWLQJWKHWHPSRUDU\WDEOHZLWKWKHÀUVWVWDWHPHQWZHLQVHUWGDWDLQWRWKHWDEOHZLWK WKHIROORZLQJINSERTVWDWHPHQW7KHQH[WVWDWHPHQWLQVHUWVWKHDSSURSULDWHGDWDLQWRWKH WDUJHWWDEOHsample.logMsgsE\VHOHFWLQJWKHDSSURSULDWHGDWDIURPWKHWHPSRUDU\GDWD WKDWPDWFKHVWKHODQJXDJHHQWU\IURPWKHconfigWDEOH7KHWHPSRUDU\WDEOHLVWKHQUHPRYHG DJDLQ7KHÀQDOSELECTVWDWHPHQWLVVROHO\IRUFKHFNLQJWKHUHVXOWVRIWKHRSHUDWLRQ
6HHDOVR f
,QVHUWLQJQHZGDWDDQGXSGDWLQJGDWDLILWDOUHDG\H[LVWV
0DQDJLQJ'DWD
'HOHWLQJDOOGDWDIURPODUJHWDEOHV $OPRVWHYHU\RQHZKRZRUNVZLWKGDWDEDVHVH[SHULHQFHVWKHFRQVWDQWJURZWKRIWKHGDWD VWRUHGLQWKHLUGDWDEDVHDQGLWLVW\SLFDOO\ZHOOEH\RQGWKHLQLWLDOHVWLPDWHV%HFDXVHRIWKDW \RXRIWHQHQGXSZLWKUDWKHUODUJHGDWDVHWV$QRWKHUFRPPRQREVHUYDWLRQLVWKDWLQPRVW GDWDEDVHVWKHUHDUHVRPHWDEOHVWKDWKDYHDVSHFLDOWHQGHQF\WRJURZHVSHFLDOO\ELJ ,IDWDEOH VVL]HUHDFKHVDYLUWXDOWKUHVKROG ZKLFKLVKDUGWRGHÀQHDVLWGHSHQGVKHDYLO\ RQWKHDFFHVVSDWWHUQVDQGWKHGDWDVWUXFWXUHV LWJHWVKDUGHUDQGKDUGHUWRPDLQWDLQDQG SHUIRUPDQFHGHJUDGDWLRQPLJKWRFFXU)URPDFHUWDLQSRLQWRQLWLVHYHQGLIÀFXOWWRJHWULG RIGDWDLQWKHWDEOHDJDLQDVWKHVKHHUQXPEHURIUHFRUGVPDNHVGHOHWLRQDSUHWW\H[SHQVLYH WDVN7KLVSDUWLFXODUO\KROGVWUXHIRUVWRUDJHHQJLQHVZLWK0XOWL9HUVLRQ&RQFXUUHQF\&RQWURO 09&& LI\RXRUGHUWKHGDWDEDVHWRGHOHWHGDWDIURPWKHWDEOHLWPXVWQRWEHGHOHWHGULJKW DZD\EHFDXVH\RXPLJKWVWLOOUROOEDFNWKHGHOHWLRQ6RHYHQZKLOHWKHGHOHWLRQZDVLQLWLDWHG DFRQFXUUHQWTXHU\RQWKHWDEOHVWLOOKDVWREHDEOHWRVHHDOOWKHUHFRUGV GHSHQGLQJRQWKH WUDQVDFWLRQLVRODWLRQOHYHO 7RDFKLHYHWKLVWKHVWRUDJHHQJLQHZLOORQO\PDUNWKHUHFRUGVDV GHOHWHGEXWWKHDFWXDOGHOHWLRQWDNHVSODFHDIWHUWKHRSHUDWLRQLVFRPPLWWHGDQGDOORWKHU WUDQVDFWLRQVWKDWDFFHVVWKLVWDEOHDUHFORVHGDVZHOO ,I\RXKDYHWRGHDOZLWKODUJHGDWDVHWVWKHPRVWGLIÀFXOWWDVNLVWRRSHUDWHRQWKHSURGXFWLRQ V\VWHPZKLOHRWKHUSURFHVVHVFRQFXUUHQWO\ZRUNRQWKHGDWD,QWKHVHFLUFXPVWDQFHV\RXKDYH WRNHHSWKHGXUDWLRQRI\RXUPDLQWHQDQFHRSHUDWLRQVDVORZDVSRVVLEOHLQRUGHUWRPLQLPL]H WKHLPSDFWRQWKHUXQQLQJV\VWHP$VWKHGHOHWLRQRIGDWDIURPDODUJHWDEOH W\SLFDOO\VWDUWLQJ DWVHYHUDOPLOOLRQVRIURZV PLJKWWDNHTXLWHVRPHWLPHWKHIROORZLQJUHFLSHVKRZVDZD\RI PLQLPL]LQJWKHGXUDWLRQRIWKLVRSHUDWLRQLQRUGHUWRUHGXFHVLGHHIIHFWV OLNHORFNLQJHIIHFWV RUSHUIRUPDQFHGHJUDGDWLRQ
*HWWLQJUHDG\ %HVLGHVDXVHUDFFRXQWZLWKDSSURSULDWHSULYLOHJHV DELETE \RXQHHGDVXIÀFLHQWO\ODUJH WDEOHWRGHOHWHGDWDIURP
)RUWKLVUHFLSHZHZLOOXVHWKHemployeesGDWDEDVHZKLFKLVDQH[DPSOH GDWDEDVHDYDLODEOHIURP0\64/ http://dev.mysql.com/doc/ employee/en/employee.html. 7KLVGDWDEDVHSURYLGHVVRPHWDEOHVZLWKVHQVLEOHGDWDDQGVRPHSUHWW\ ODUJHWDEOHVWKHODUJHVWKDYLQJPRUHWKDQPLOOLRQUHFRUGV
:HDVVXPHWKDWWKHEmployeesGDWDEDVHZDVLQVWDOOHGZLWKDQ,QQR'%VWRUDJHHQJLQH HQDEOHG7RGHOHWHDOOURZVRIWKHODUJHVWWDEOHemployees.salariesLQDTXLFNZD\SOHDVH UHDGRQ
&KDSWHU
+RZWRGRLW &RQQHFWWR\RXUGDWDEDVH (QWHUWKHIROORZLQJ64/FRPPDQG mysql> TRUNCATE TABLE employees.salaries; Query OK, 0 rows affected (0.16 sec)
+RZLWZRUNV 7KHTRUNCATE TABLEFRPPDQGLVDUDWKHUIDVWZD\RIGHOHWLQJDOOGDWDIURPDWDEOH)RU WDEOHVWKDWDUHQRWUHIHUHQFHGE\)RUHLJQNH\FRQVWUDLQWV PRUHRQWKDWODWHU WKHFRPPDQG EDVLFDOO\GURSVWKHWDEOHWHPSRUDULO\DQGUHFUHDWHVWKHWDEOHZLWKWKHVDPHVWUXFWXUHDV EHIRUH7KLVRSHUDWLRQKDVEDVLFDOO\DFRQVWDQWWLPHFKDUDFWHULVWLF³WKHDPRXQWRIGDWDVWRUHG LQVLGHWKHWDEOHGRHVQRWKDYHDQ\HIIHFWLQWKHWLPHQHHGHGIRUWKHTRUNCATEFRPPDQG %HIRUH0\64/WKHTRUNCATE TABLEVWDWHPHQWIRU,QQR'%WDEOHV ZDVDOZD\VHTXLYDOHQWWRDDELETEVWDWHPHQWUHJDUGOHVVRIZKHWKHU)RUHLJQ NH\FRQVWUDLQWVH[LVWRUQRW7RWDNHDGYDQWDJHRIWKHVSHHGLPSURYHPHQWV \RXKDYHWRXVH0\64/RUODWHU
,QFRPSDULVRQWRDFODVVLFDODELETE FROM employees.salaries;RSHUDWLRQWKH UHGXFWLRQLQWLPHQHHGHGLVVWULNLQJ 2SHUDWLRQ
7LPHQHHGHG
7581&$7(7$%/(
VHF
'(/(7(
PLQVHF
100
Time in seconds
80
60
40
20
0 TRUNCATETABLE
DELETE
0DQDJLQJ'DWD 7KHTRUNCATE TABLEFRPPDQGWDNHVRQO\DIUDFWLRQRIWKHWLPHQHHGHGIRUWKHDELETE +RZHYHUWKHUHDUHVRPHFDYHDWV )LUVWRIDOOWKHTRUNCATEFRPPDQGZLOORQO\KDYHWKHVSHHGDGYDQWDJHRQ,QQR'%WDEOHV LIWKHWDEOHLVQRWUHIHUHQFHGE\DQ\)RUHLJQNH\FRQVWUDLQWV%XWLIWKHWDEOHLVUHIHUHQFHG E\)RUHLJQNH\VWKHTRUNCATE TABLEFRPPDQGLVHTXLYDOHQWWRH[HFXWLQJDDELETE VWDWHPHQWZLWKQRWHEREFODXVHDOVRHOLPLQDWLQJDOOVSHHGGLIIHUHQFHV mysql> CREATE TABLE employees.salaries_referencer ( -> emp_no
INT,
-> from_date
DATE,
-> CONSTRAINT
salaries_fk
-> FOREIGN KEY (emp_no, from_date) -> REFERENCES
salaries (emp_no, from_date)
-> ON DELETE RESTRICT); Query OK, 0 rows affected (0.08 sec)
mysql> TRUNCATE TABLE employees.salaries; Query OK, 0 rows affected (1 min 33.44 sec)
2SHUDWLRQ
7LPHQHHGHG
7581&$7(7$%/( ZLWKIRUHLJQNH\UHI
PLQVHF
'(/(7(
PLQ VHF
100
Time in seconds
80 60 40 20 0 TRUNCATETABLE (w/FKreferences)
DELETE
)XUWKHUPRUHWKHTRUNCATEVWDWHPHQWUHTXLUHVWKHDROPSULYLOHJH EHIRUH0\64/ LWRQO\UHTXLUHVWKHDELETESHUPLVVLRQ ZKLFKIRUELGVXVHRIWKLVFRPPDQGIRUVRPHXVHUV
&KDSWHU
$QGÀQDOO\TRUNCATELVQRWDWUDQVDFWLRQVDIHFRPPDQG,I\RXH[HFXWHDTRUNCATE VWDWHPHQW\RXZLOOQRWEHDEOHWRSHUIRUPDUROOEDFNRQWKLVRSHUDWLRQDQ\PRUHDQGDQ\ RSHQRSHUDWLRQIURPWKHFXUUHQWWUDQVDFWLRQJHWVDXWRPDWLFDOO\FRPPLWWHGDVZHOO7KLV LVDFKDUDFWHULVWLFWKDWGLVTXDOLÀHVWKLVVWDWHPHQWIRUVLWXDWLRQVLQZKLFKWKHSRVVLELOLW\RI SHUIRUPLQJDUROOEDFNLVPDQGDWRU\ \RXZLOOKDYHWRVWLFNZLWKWKH PXFKVORZHU DELETELQ WKHVHFDVHV
7KHUH VPRUH $VZHKDYHVHHQTRUNCATE TABLERQO\KDVSHUIRUPDQFHDGYDQWDJHVLIWKHUHLVQR)RUHLJQ NH\UHIHUHQFHWRWKHWDEOHWKDWLVWREHGHOHWHG+HUHZHZLOOGLVFXVVKRZWRXVHWKHVSHHG LPSURYHPHQWVHYHQLQFDVHRIH[LVWLQJUHIHUHQFHV
7HPSRUDULO\GLVDEOLQJ)RUHLJQNH\FRQVWUDLQWV 7RPDNHXVHRIWKHLQFUHDVHGVSHHGRITRUNCATE TABLEDOWKRXJKWKHWDUJHWWDEOHLVUHI HUHQFHGYLD)RUHLJQNH\V\RXFRXOGWHPSRUDULO\UHPRYHWKH)RUHLJQNH\FRQVWUDLQWVXVHWKH TRUNCATE TABLEFRPPDQGDQGUHHVWDEOLVKWKHUHIHUHQFHVDIWHUZDUGV8VLQJWKHDERYH H[DPSOHRIDWDEOHsalaries_referencerWKDWUHIHUHQFHVVDODULHV\RXFRXOGXVHWKH IROORZLQJVHTXHQFH mysql> ALTER TABLE employees.salaries_referencer -> DROP FOREIGN KEY salaries_fk; Query OK, 0 rows affected (0.19 sec) Records: 0
Duplicates: 0
Warnings: 0
mysql> TRUNCATE TABLE employees.salaries; Query OK, 0 rows affected (0.44 sec)
mysql> ALTER TABLE employees.salaries_referencer -> ADD CONSTRAINT salaries_fk -> FOREIGN KEY (emp_no, from_date) -> REFERENCES salaries (emp_no, from_date) -> ON DELETE RESTRICT; Query OK, 0 rows affected (0.14 sec) Records: 0
Duplicates: 0
Warnings: 0
0DQDJLQJ'DWD :LWKWKLVVHTXHQFH\RXWHPSRUDULO\GLVDEOHWKH)RUHLJQNH\FRQVWUDLQWVWRKDYHTRUNCATE TABLEXVHWKHIDVWHUGHOHWLRQPHWKRG%HZDUHKRZHYHUWKDWWKLVPHWKRGPLJKWDOVR OHDGWRSUREOHPVZKHQWKHGHOHWLRQRIWKHWDEOHSURGXFHVORRVHHQGV,IWKHUHIHUHQFLQJ WDEOHsalaries.referencerKROGVUHFRUGVWKDWUHIHUHQFHGWKHQRZHPSW\WDUJHWWDEOH salariesWKHFUHDWLRQRIWKH)RUHLJQNH\FRQVWUDLQWVZLOOIDLO
$OVRNHHSLQPLQGWKDWWKLVVLWXDWLRQPLJKWDOVRRFFXUEHFDXVHRIFRQFXUUHQWSURFHVVHV ZKLFKDUHDEOH IRUWKHGXUDWLRQRIWKHGLVDEOHGFRQVWUDLQWV WRLQVHUWGDWDLQWRWKHWDEOHVWKDW YLRODWHWKHLQWHQGHGUHIHUHQWLDOLQWHJULW\ $VDQDOWHUQDWLYH\RXPLJKWEHWHPSWHGWRWHPSRUDULO\GLVDEOHWKH)RUHLJQNH\FKHFNVE\ VHWWLQJforeign_key_checksWR]HUR:KLOHWKLVZRUNVUHJDUGLQJWKHTRUNCATE TABLE SHUIRUPDQFHLWLVVWURQJO\GLVFRXUDJHGWRXVHWKLVRSWLRQEHFDXVHWKH)RUHLJQNH\LQWHJULW\LV QRWUHYDOLGDWHGZKHQWKH)RUHLJQNH\FKHFNVDUHHQDEOHGDJDLQ6R\RXULVNLQFRQVLVWHQWGDWD ZLWKUHVSHFWWRWKHUHIHUHQWLDOLQWHJULW\
'HOHWLQJDOOEXWDIUDJPHQWRIDODUJH WDEOH VGDWD ,QWKHSUHYLRXVUHFLSH'HOHWLQJDOOGDWDIURPODUJHWDEOHVZHGLVFXVVHGDPHWKRGRITXLFNO\ UHPRYLQJDOOGDWDIURPODUJHWDEOHVZKLOHDYRLGLQJSHUIRUPDQFHKLWV%XWH[SHULHQFHVKRZV WKDW\RXRIWHQPXVWQRWGHOHWHDOOGDWDEXWKDYHWRUHWDLQVRPHUHFRUGVDQGGHOHWHWKHUHVW 7KHTRUNCATE TABLEFRPPDQGGRHVQRWDOORZDQ\DGGLWLRQDOFODXVHVWRGHÀQHZKLFK UHFRUGVWRGHOHWHDQGZKLFKQRW LWDOZD\VGHOHWHVDOOHQWULHV 7KHLQWXLWLYHVROXWLRQWRWKLVZRXOGEHWRXVHDQRUPDODELETEFRPPDQGZLWKDWHEREFODXVH WKDWRQO\PDWFKHVWKHUHFRUGVWRGHOHWH)RUODUJHWDEOHVWKLVPLJKWSURYHTXLWHDQH[SHQVLYH RSHUDWLRQ LQWHUPVRIGXUDWLRQ ,QWKLVUHFLSHZHZLOOVKRZ\RXKRZWRTXLFNO\UHPRYHPRVWRI WKHGDWDIURPODUJHWDEOHVZKLOHSUHVHUYLQJVRPHRIWKHUHFRUGV
&KDSWHU
*HWWLQJUHDG\ :HDJDLQQHHGD0\64/VHUYHUXSDQGUXQQLQJDQGD64/FOLHQW OLNHmysql )RUWKLVUHFLSH ZHDOVRQHHGDXVHUDFFRXQWZLWKSELECTINSERTDELETEDROPDQGCREATESULYLOHJHV IRUWKHWDUJHWGDWDEDVH ZHZLOOXVHWKHsample_installXVHUWKURXJKRXWWKLVVHFWLRQ :H ZLOOIXUWKHUPRUHXVHWKH(PSOR\HHVVDPSOHGDWDEDVHLQDQ,QQR'%FRQWH[W7KLVGDWDEDVH ZDVLQWURGXFHGLQWKHSUHYLRXVUHFLSHDQGLVDYDLODEOHIRUIUHHRQWKH0\64/ZHEVLWH:HZLOO XVHWKHODUJHVWWDEOHsalaries ZLWKPRUHWKDQPLOOLRQUHFRUGV DVWKHWDEOHWRGHOHWH IURP,QRXUH[DPSOHZHZLOOGHOHWHDOOUHFRUGVKDYLQJDfrom_dateEHIRUHWKHWKUHVKROGRI '2002-01-01 00:00:00.0'
+RZWRGRLW &RQQHFWWRWKHGDWDEDVHXVLQJD64/FOLHQWDQGWKHsample_installDFFRXQW ([HFXWHWKHIROORZLQJFRPPDQGV mysql> use employees; Database changed mysql> CREATE TABLE salaries_part -> SELECT * FROM salaries -> WHERE from_date >= "2002-01-01 00:00:00.0"; Query OK, 140930 rows affected (11.47 sec)Records: 140930
Duplicates: 0
Warnings: 0
mysql> TRUNCATE TABLE salaries; Query OK, 0 rows affected (0.05 sec)
mysql> INSERT INTO salaries SELECT * from salaries_part; Query OK, 140930 rows affected (4.63 sec) Records: 140930
Duplicates: 0
Warnings: 0
mysql> DROP TABLE salaries_part; Query OK, 0 rows affected (0.06 sec)
0DQDJLQJ'DWD
+RZLWZRUNV )RUVSHHGLQJXSWKHGHOHWLRQRIPRVWRIWKHGDWDIURPDODUJHWDEOHZHXWLOL]HWKHVSHHG DGYDQWDJHRIWKHTRUNCATE TABLEVWDWHPHQWRYHUDDELETEFRPPDQG,QGHWDLOWKHVWHSV DUHDVIROORZV 7KHLQLWLDOUSEVWDWHPHQWLVIRUFRQYHQLHQFHRQO\VRZHGRQRWKDYHWRJLYHWKHemployees SUHÀ[IRUHYHU\WDEOH :LWKWKHQH[WVWDWHPHQW CREATE TABLE … SELECT * FROM … ZHVLPSO\FRS\WKHGDWD WKDWVKRXOGQRWEHUHPRYHGWRDQHZO\FUHDWHGWDEOHsalaries_part %HFDUHIXOWRDYRLGHUURUVZKHQLQYHUWLQJFRQGLWLRQV WRGHOHWHDOOHQWULHV EHIRUHWLPH;\RXKDYHWRFRS\DOOUHFRUGVYDOXHVODWHURUHTXDOWR;,I \RXFRS\RQO\UHFRUGVODWHUWKDQ;DOOUHFRUGVH[DFWO\DWWLPH;ZRXOGJHW GHOHWHGDVZHOO
7KLVWDEOHWHPSRUDULO\KROGVWKHGDWDZKLOHZHGHOHWHDOOGDWDIURPWKHODUJHsalaries WDEOHXVLQJTRUNCATELQWKHQH[WVWHS$IWHUZDUGVZHVLPSO\FRS\WKHSDUWLDOGDWDIURPWKH salaries_partWDEOHEDFNLQWRWKHRULJLQDO QRZHPSWLHG salariesWDEOH:LWKWKHÀQDO VWHSZHVFUDSWKHsalaries_partWDEOHDJDLQDVLWLVQRWQHHGHGDQ\PRUH SHOW VARIABLES LIKE 'innodb_fast_shutdown';
/RJRIIIURP0\64/DQGVWRSWKH0\64/VHUYHU /RFDWHWKH0\64/FRQÀJXUDWLRQÀOHXVXDOO\FDOOHGmy.cnfRUmy.ini RQ:LQGRZV DQGRSHQLWLQDWH[WHGLWRU /RFDWHWKHIROORZLQJSDUDPHWHUVLQWKH[mysqld]VHFWLRQ \RXYDOXHVZLOOYDU\ RIFRXUVH [mysqld] ... innodb_log_group_home_dir=/var/lib/mysql/redolog innodb_log_file_size=32M innodb_log_buffer_size=64M innodb_log_files_in_group=2 ...
(GLWWKHDERYHFRQÀJXUDWLRQVHWWLQJVWRWKHLUQHZYDOXHV,I\RXUHTXLUHKHOSRQKRZWR ÀQGVXLWDEOHYDOXHVVHHWKH7KHUH VPRUHVHFWLRQRIWKLVUHFLSH
6DYHWKHFRQÀJXUDWLRQÀOH 1DYLJDWHWRWKHGLUHFWRU\FRQÀJXUHGIRUinnodb_log_group_home_dir,IWKHUHLV QRVXFKVHWWLQJLQ\RXUFRQÀJXUDWLRQÀOHQDYLJDWHWR0\64/ VGDWDGLUHFWRU\WKDWLV WKHQWDNHQDVWKHGHIDXOW
&RQÀJXULQJ0\64/ 0RYHWKHÀOHVZKRVHQDPHVVWDUWZLWKib_logfileWRDEDFNXSORFDWLRQ'RQRW FRS\WKHP WKH\PXVWEHUHPRYHGIURPWKHLURULJLQDOORFDWLRQ 5HVWDUWWKH0\64/VHUYHU 9HULI\WKDWQHZÀOHVDUHFUHDWHGDV\RXFRQÀJXUHGWKHP $ ls -l /var/lib/mysqld/redolog
,I\RXGRQRWVHHWKHQHZÀOHVDSSHDUDQGWKHVHUYHUGRHVQRWVWDUWXSFRUUHFWO\FKHFNWKH 0\64/HUURUORJIRUPHVVDJHV8VXDOO\WKHRQO\WKLQJWKDWFDQJRZURQJKHUHLVWKDW\RX HLWKHUPLVW\SHGWKHGLUHFWRU\QDPHRUGLGQRWDFWXDOO\UHPRYHWKHSUHYLRXVib_logfileÀOHV 7RUHVWRUHHYHU\WKLQJEDFNWRWKHRULJLQDOFRQÀJXUDWLRQUHVWRUH\RXUFRQÀJXUDWLRQÀOHIURPWKH EDFNXSDQGUHVWRUHWKHib_logfileÀOHV\RXPRYHGRXWWRWKHEDFNXSWRWKHLURULJLQDOORFDWLRQ
:KDWMXVWKDSSHQHG %\VHWWLQJinnodb_fast_shutdownWR0\RXWROGWKHVHUYHUWRÀQLVKZULWLQJDQ\SHQGLQJ FKDQJHVWRWKHGLVNEHIRUHDFWXDOO\H[LWLQJ7KLVPDNHVVXUHWKHUHDUHQRUHPDLQLQJ WUDQVDFWLRQVLQWKHFXUUHQWUHGRORJVWKDWFRXOGJHWORVWZKHQWKHVHÀOHVDUHUHSODFHG $IWHUWKDW\RXFRXOGFKDQJHWKHFRQÀJXUDWLRQWRQHZYDOXHVSRVVLEO\XVLQJDGLIIHUHQWQXPEHU RIÀOHVDQGGLIIHUHQWVL]HV 7KHQEHIRUHUHVWDUWLQJ\RXFRXOGPRYHWKHROGUHGRORJÀOHVRXWRIWKHZD\7KLVLVLPSRUWDQW EHFDXVHRWKHUZLVH0\64/ZRXOGFRPSODLQDERXWDPLVPDWFKEHWZHHQWKHVHWWLQJVÀOHDQG WKHDFWXDOVLWXDWLRQRQGLVN:KHQLWFRPHVXSÀQGLQJQRUHGRORJÀOHVLWZLOOFUHDWHQHZRQHV ZLWKWKHVHWWLQJVMXVWFRQÀJXUHG
7KHUH VPRUH 2IWHQZKHQWDONLQJDERXWWUDQVDFWLRQVWKHZRUGUROOEDFNFRPHVXS,WPHDQVWKDWLIVRPHWKLQJ JRHVZURQJLQWKHPLGGOHRIDSRVVLEO\FRPSOH[GDWDPDQLSXODWLRQRSHUDWLRQDQGLWKDVWREH DERUWHGWKHGDWDEDVHVHUYHUZLOOVDIHO\UHVWRUHHYHU\WKLQJEDFNWRWKHVWDWHLWZDVLQZKHQWKDW RSHUDWLRQEHJDQQRWOHDYLQJDQ\GDWDRQO\SDUWLDOO\GHOHWHGRUPRGLÀHG
&KDSWHU
7KHRSSRVLWHWHUP³UROOIRUZDUGRUUHGRLQJ³LVOHVVFRPPRQO\XVHG,WPHDQVWKDWZKHQHYHU WKHFRPSOH[RSHUDWLRQPHQWLRQHGHDUOLHUFRPSOHWHVVXFFHVVIXOO\\RXDUHJXDUDQWHHGWKDW QRWKLQJVKRUWRIDFWXDOKDUGZDUHIDLOXUHFRXOGOHDGWRWKHVHFKDQJHVEHLQJORVWDJDLQ 7KLVPLJKWDSSHDUREYLRXVEHFDXVHRQHZRXOGH[SHFWWKHGDWDEDVHVHUYHUQRWWRUHSRUW DQ\WKLQJDVVXFFHVVIXOXQOHVVLWZDVDFWXDOO\FRPSOHWHO\GRQH+RZHYHULIWKDWZHUHVRZULWH RSHUDWLRQVZRXOGEHFRPHSDLQIXOO\VORZDVWKHXQGHUO\LQJ,2VXEV\VWHP JHQHUDOO\PHDQLQJ KDUGGLVNV LVYHU\RIWHQWKHERWWOHQHFNFRPSRQHQW 7RHYDGHWKLVSRWHQWLDOSHUIRUPDQFHSUREOHPPRVWWUDQVDFWLRQDOGDWDEDVHV,QQR'%EHLQJ QRGLIIHUHQWXVHVWKHFRQFHSWRIDWUDQVDFWLRQ³RUUHGR³ORJWKDWDOORZVLWWRPRUHHIÀFLHQWO\ KDQGOHZULWHRSHUDWLRQVZLWKRXWULVNLQJGDWDLQWHJULW\7KHUHGRORJZRUNVDVDVRUWRIVFUDWFK SDGFRQWDLQLQJLQIRUPDWLRQRQZKDWUHPDLQVWREHGRQHWRWKHGDWDÀOHV:LWKLWVKHOSWKH VHUYHUFDQRSWLPL]HGLVNDFFHVVWRLPSURYHSHUIRUPDQFH 7KHLGHDOVL]HIRUWKHUHGRORJGHSHQGVRQWKHVL]HDQGQXPEHURIWUDQVDFWLRQVWKHVHUYHUKDV WRSURFHVV*HQHUDOO\VSHDNLQJWKHORJVKRXOGEHODUJHHQRXJKWRVWRUHDQ\VLQJOHWUDQVDFWLRQ SOXVDERXWSHUFHQW$VDUXOHRIWKXPEWKHWRWDOORJVL]H WKHQXPEHURIORJÀOHVWLPHVWKHLU LQGLYLGXDOVL]H QHHGQRWH[FHHGDERXWRIWKH,QQR'%EXIIHUSRROVL]H )RUPRUHLQIRUPDWLRQRQUHGRORJVDQGKRZWRGHWHUPLQHDVHQVLEOHVL]HVHWWLQJYLVLW http://mysqldump.azundris.com/archives/78-Configuring-InnoDB-AnInnoDB-tutorial.htmlIRUDGHWDLOHGGHVFULSWLRQRI,QQR'%ORJFRQÀJXUDWLRQ
8QGHUVWDQGLQJDQGFRQÀJXULQJLPSRUWDQW 0\64/DQG,QQR'%WLPHRXWRSWLRQV 0\64/ VFRQÀJXUDWLRQÀOHFDQFRQWDLQDYDULHW\RIGLIIHUHQWWLPHRXWVHWWLQJVHDFKUHVSRQVLEOH IRUDVSHFLÀFNLQGRIRSHUDWLRQRUFRQQHFWLRQ,QWKLVUHFLSHZHSUHVHQWDVHOHFWLRQRIWKHVH WLPHRXWVHWWLQJVDQGDVXJJHVWHGYDOXHWRJRDORQJZLWKHDFK7KH+RZLWZRUNVVHFWLRQKDV GHWDLOVRQHDFKYDOXHSUHVHQWHG ,QJHQHUDOWKHYDOXHVVXJJHVWHGKHUHVKRXOGEHDSSURSULDWHIRUERWK0\64/ YHUVLRQVDQG+RZHYHUSOHDVHQRWHWKDWDQ\RIWKHVHRSWLRQVPD\ ZHOOYDU\IRU\RXUHQYLURQPHQWGHSHQGLQJRQZKDWWKHUHTXLUHPHQWVDUH VR SOHDVHGRQRWVLPSO\XVHWKHVHYDOXHVYHUEDWLP
*HWWLQJUHDG\ 7RDSSO\WLPHRXWFRQÀJXUDWLRQVHWWLQJV\RXZLOOQHHGDFFHVVWRWKH0\64/FRQÀJXUDWLRQ ÀOH³W\SLFDOO\my.cnfRUmy.ini RQ:LQGRZV ³DQGWKHULJKWVWRUHVWDUWWKHVHUYHUWRKDYH DQ\FKDQJHVPDGHWRWKHFRQÀJXUDWLRQWDNHHIIHFW
&RQÀJXULQJ0\64/
+RZWRGRLW /RFDWHWKH0\64/FRQÀJXUDWLRQÀOHDQGRSHQLWLQDWH[WHGLWRU ,QWKH[mysqld]VHFWLRQVHWXSWKHIROORZLQJYDOXHV6RPHRIWKHRSWLRQVPD\ DOUHDG\EHSUHVHQW RWKHUVZLOOOLNHO\KDYHWREHDGGHG0DNHVXUHHDFKRSWLRQRQO\ DSSHDUVRQFH [mysqld] ...innodb_rollback_on_timeout innodb_lock_wait_timeout=50 interactive_timeout=1200 wait_timeout=28800 net_read_timeout=30 net_write_timeout=120 ...
6DYHWKHÀOH 5HVWDUWWKH0\64/VHUYHU
+RZLWZRUNV %\VHWWLQJWKHYDOXHVDVGHVFULEHGLQWKHDERYHVHFWLRQ\RXWHOO0\64/WRXVHGLIIHUHQWYDOXHV WKDQWKHGHIDXOWVIRUWKHRSWLRQVPHQWLRQHG7KHQHZVHWWLQJVWDNHHIIHFWZLWKWKHVHUYHUUHVWDUW
6HWWLQJYDOXHVIRULQQRGEBUROOEDFNBRQBWLPHRXWLQQRGEBORFNBZDLWB WLPHRXW 7KHVHWZRVHWWLQJVDUHSUREDEO\WKHPRVWLPSRUWDQWDVUHJDUGVWKHORFNLQJEHKDYLRULQDQ\ 0\64/VHWXS6WDUWLQJZLWKYHUVLRQ0\64/FKDQJHGWKHUROOEDFNEHKDYLRULQFDVH DWLPHRXWRFFXUUHGEHFDXVHDWUDQVDFWLRQFRXOGQRWDFTXLUHDORFNIRUDURZ7KLVXVXDOO\ KDSSHQVZKHQDQRWKHUWUDQVDFWLRQLVVWLOOZRUNLQJRQWKDWURZDQGWKLVLVWREHH[SHFWHG LQQRUPDOGDWDEDVHRSHUDWLRQV7KHGDWDEDVHVHUYHUUROOVEDFNWKHHQWLUHWUDQVDFWLRQLQ WKLVFDVH$SSOLFDWLRQVVKRXOGEHGHVLJQHGWRUHVSRQGWRVXFKFRQGLWLRQVE\UHWU\LQJWKH HQWLUHWUDQVDFWLRQ 3ULRUWRWKLVZDVWKHGHIDXOWEHKDYLRUPDLQWDLQLQJWKHUXOHWKDWDWUDQVDFWLRQHLWKHU VXFFHHGVRUIDLOVDQGLVUROOHGEDFNFRPSOHWHO\DVDQDWRPLFHQWLW\,QDQGQHZHU YHUVLRQVWKHGHIDXOWZDVPRGLÀHGWRUROOEDFNRQO\WKHYHU\ODVWVWDWHPHQWRIWKHIDLOLQJ WUDQVDFWLRQLQVWHDGNHHSLQJWKHWUDQVDFWLRQRSHQ:KLOHWKHUHLVDUHDVRQEHKLQGWKDWFKDQJH IRUORQJWUDQVDFWLRQVLWFDQEHIDVWHUWRMXVWUHWU\WKHYHU\ODVWVWDWHPHQWWKDQZDLWIRUWKH UROOEDFNDQGWU\DJDLQIURPWKHVWDUW LWUHTXLUHVVSHFLDOSUHFDXWLRQVWDNHQRQWKHDSSOLFDWLRQ
&KDSWHU
OHYHOWREHDEOHWRKDQGOHWKLVYHU\0\64/VSHFLÀFVFHQDULR
6HWWLQJWKHinnodb_rollback_on_timeoutRSWLRQLQWKHFRQÀJXUDWLRQ ÀOHUHVWRUHVWKHPRUHVWDQGDUGZD\RIUROOLQJEDFNWKHZKROHWUDQVDFWLRQLQ FDVHRIDORFNZDLWWLPHRXW:HVWURQJO\UHFRPPHQGHQDEOLQJWKLVRSWLRQ XQOHVV\RXDUHSHUIHFWO\VXUH\RXUDSSOLFDWLRQLVDZDUHRIWKH0\64/VSHFLÀF EHKDYLRUFRQÀJXUHGDVWKHGHIDXOW innodb_lock_wait_timeoutGHÀQHVWKHQXPEHURIVHFRQGVDWUDQVDFWLRQZLOOZDLWWR
DFTXLUHDQHFHVVDU\ORFNZKHQDVHFRQGWUDQVDFWLRQLVZRUNLQJFRQFXUUHQWO\RQWKHVDPH UHFRUGV7KHGHIDXOWYDOXHLVVHFRQGVDQGLIWKHORFNFRXOGQRWEHDFTXLUHGE\WKHQD WLPHRXWHUURUZLOORFFXUDQGWKHWUDQVDFWLRQZLOOEHUROOHGEDFN'HSHQGLQJRQKRZORQJWKH WUDQVDFWLRQVLQ\RXUVHWXSW\SLFDOO\WDNHWKLVYDOXHRIWHQQHHGVWREHDGMXVWHG,I\RXUHJXODUO\ KDYHEXONGDWDRSHUDWLRQVWKDWDIIHFWDORWRIURZV\RXZLOOZDQWWRLQFUHDVHWKLVYDOXH,IRQ WKHRWKHUKDQG\RXUV\VWHPQRUPDOO\XVHVYHU\VKRUWWUDQVDFWLRQVUHGXFLQJWKLVYDOXHPD\ KHOS\RXÀQGRXWDERXWSUREOHPVZLWKORFNFRQWHQWLRQHDUOLHU
6HWWLQJYDOXHVIRULQWHUDFWLYHBWLPHRXWZDLWBWLPHRXW interactive_timeoutGHÀQHVKRZORQJDQLQWHUDFWLYHFOLHQWFRQQHFWLRQFDQEHLGOH
EHIRUHWKHVHUYHUFORVHVLWDXWRPDWLFDOO\ ,GOH LQWKLVFRQWH[WUHIHUVWRWKHWLPHEHWZHHQWZR VWDWHPHQWVEHLQJH[HFXWHGZLWKQRDFWLYLW\LQEHWZHHQ :HUHFRPPHQGUHGXFLQJWKLVIURPWKHGHIDXOWYDOXHRIVHFRQGV KRXUV WRDPXFK ORZHUYDOXHOLNHVHFRQGV PLQXWHV 7KLVDOORZVWKHVHUYHUWRFORVHLGOHFRQQHFWLRQV DQGFRQVHUYHVRPHUHVRXUFHV 7KHFRXQWHUSDUWYDULDEOHIRUQRQLQWHUDFWLYHVHVVLRQVVXFKDVWKRVHIURPDQDSSOLFDWLRQ VHUYHU VFRQQHFWLRQSRROLVFDOOHGwait_timeoutDQGKDVWKHVDPHVHPDQWLFV'HSHQGLQJ RQ\RXUDSSOLFDWLRQ\RXPLJKWZDQWWROHDYHWKLVVHWWLQJRQDKLJKHUYDOXHDVPRVWFRQQHFWLRQ SRROVFDQEHFRQÀJXUHGWRUHOHDVHFRQQHFWLRQVDXWRPDWLFDOO\GHSHQGLQJRQFXUUHQW ORDGFRQGLWLRQV
6HWWLQJYDOXHVIRUQHWBUHDGBWLPHRXWQHWBZULWHBWLPHRXW 7KHSURWRFRO0\64/XVHVWRKDQGOHFRPPXQLFDWLRQEHWZHHQVHUYHUDQGFOLHQWVLVUDWKHU OLPLWHGLQGHVLJQDOORZLQJRQO\RQHRSHUDWLRQWREHFDUULHGRXWDWDWLPH$VLGHHIIHFWRIWKLVLV WKDWRQFHDGDWDWUDQVIHULQHLWKHUGLUHFWLRQKDVVWDUWHGWKHUHLVQRZD\IRULWWREHLQWHUUXSWHG LQDFRQWUROOHGPDQQHU 7KHnet_read_timeoutFRQWUROVKRZORQJDSLHFHRILQIRUPDWLRQFDQEHVHQWIURPWKHFOLHQW WRWKHVHUYHUEHIRUHWKHFRQQHFWLRQLVDERUWHG7KLVLVXVXDOO\QRWDSUREOHP³WKHGHIDXOW VHWWLQJLVVHFRQGV8QGHUQRUHJXODUFLUFXPVWDQFHVZLOODFRPPXQLFDWLRQLQWKDWGLUHFWLRQ WDNHVRORQJ
&RQÀJXULQJ0\64/ 7KHnet_write_timeoutLVPRUHSUREOHPDWLFEHFDXVHIRUODUJHUHVXOWVHWVWKHGHIDXOW YDOXHRIVHFRQGVPLJKWEHWRRVKRUW7KLVLVHVSHFLDOO\WUXHIRUFOLHQWVWKDWIHWFKWKHUHVXOW LQVWUHDPLQJPRGHSRWHQWLDOO\SHUIRUPLQJWLPHFRQVXPLQJRSHUDWLRQVRQHDFKURZUHWULHYHG WKHUHE\PDNLQJWKHRYHUDOORSHUDWLRQWDNHORQJHUWKDQWKHWLPHRXW 7KHH[DFWYDOXHUHTXLUHGIRU\RXUVHWXSGHSHQGVRQKRZFOLHQWVIHWFKGDWDDQGZRUNZLWKLW VR \RXZLOOQHHGWRH[SHULPHQWDQGÀQGDVXLWDEOHYDOXH 2QHFDYHDWWRFRQVLGHUZLWKnet_write_timeoutLVWKDWLWPD\OHDGWRVHHPLQJO\UDQGRP IDLOXUHVRImysqldumpOLNHWKLV mysqldump: Error 2013: Lost connection to MySQL server during query when dumping table `tablename` at row: 935578
7KLVFDQKDSSHQLIWKHIROORZLQJFRQGLWLRQVDSSO\ f
net_write_timeoutLVVHWWRDORZYDOXH
f
max_allowed_packetLVVHWWRDODUJHYDOXH
'HSHQGLQJRQWKHVSHHGRIWKHQHWZRUNRYHUZKLFKmysqldumpKDVFRQQHFWHGDQGWKHVL]HRI WKHURZVEHLQJGXPSHGLWPD\EHQHFHVVDU\WRLQFUHDVHnet_write_timeoutPXFKKLJKHU DWOHDVWWRDVORQJDVLWWDNHVWRWUDQVPLWmax_allowed_packetE\WHVRYHUWKHQHWZRUNDQG ZULWHLWWRWKHRXWSXW mysqldumpLVDUHJXODUFOLHQWSURJUDPDQGVXEMHFWWRWKHnet_write_timeout VHWWLQJ:KHQWKHVHUYHUVHQGVURZVWREHGXPSHGWRmysqldumpLQFKXQNVRIXS WRmax_allowed_packetE\WHVGHSHQGLQJRQWKHQHWZRUNFRQQHFWLRQLQEHWZHHQ WKLVPLJKWWDNHORQJHUWKDQnet_write_timeoutDOORZVPDNLQJWKHVHUYHUFXWWKH FRQQHFWLRQHYHQWKRXJKQRWKLQJLVUHDOO\ZURQJ,QFUHDVLQJnet_write_timeout IRUWKHmysqldumpWRRO VVHVVLRQZRXOGUHPHG\WKLVEXWXQIRUWXQDWHO\DVRIWKH WLPHRIZULWLQJWKHUHLVQRVXFKVHWWLQJIRUmysqldump$ZRUNDURXQGLI\RXHQFRXQWHU WKLVSUREOHPLVWRWHPSRUDULO\LQFUHDVHWKHJOREDOVHUYHUnet_write_timeoutYDOXH $ mysql -uroot -e "SELECT @@GLOBAL.net_write_timeout AS oldvalue; SET GLOBAL net_write_timeout=600;" $P\VTOGXPS $ mysql -e "SET GLOBAL net_write_timeout=oldvalue;"
7KHÀUVWFRPPDQGZLOOGLVSOD\WKHFXUUHQWYDOXHIRUnet_write_timeoutDQGWKHQVHWLW WRPLQXWHV$IWHUWKDWWKHmysqldumpFDQWDNHSODFH)LQDOO\WKHROGYDOXHLVUHVHW MXVW PDNHVXUH\RXÀOOLQWKHFRUUHFWROGYDOXH 6HH0\64/%XJDWhttp://bugs.mysql.com/bug.php?id=46103IRU PRUHGHWDLOV
&KDSWHU
$GMXVWLQJWDEOHDQGGDWDEDVHQDPHOHWWHU FDVHKDQGOLQJIRUEHWWHUSODWIRUP LQGHSHQGHQFH 0\64/LVDYDLODEOHIRUDYDULHW\RISODWIRUPV³WKHPDMRURQHVEHLQJ:LQGRZVDQG/LQX[(YHQ WKRXJKGDWDÀOHVDUHFRPSDWLEOHDQGFDQEHWUDQVIHUUHGEHWZHHQSODWIRUPVDQGFRQÀJXUDWLRQ PRVWO\IROORZVWKHVDPHJHQHUDOSULQFLSOHVWKHUHLVDQLPSRUWDQWFDYHDWWRNQRZDERXWKRZ GLIIHUHQWRSHUDWLQJV\VWHPVKDQGOHÀOHQDPHV ,QWKLVUHFLSHZHZLOOVKRZ\RXKRZWRVHWXS0\64/LQDZD\VXFKWKDWLWLVPXFKOHVVOLNHO\WR UXQLQWRSUREOHPVZKHQPRYLQJGDWDÀOHVEHWZHHQSODWIRUPV%HFDXVH0\64/GDWDEDVHVDQG WDEOHVFRUUHODWHWRÀOHV\VWHPREMHFWV GLUHFWRULHVDQGÀOHV GLIIHUHQFHVLQKRZWKHRSHUDWLQJ V\VWHP RUUDWKHUWKHÀOHV\VWHP KDQGOHVÀOHDQGGLUHFWRU\QDPHVFDQOHDGWRXQGHVLUHG HIIHFWVHVSHFLDOO\ZKHQZRUNLQJLQKHWHURJHQRXVHQYLURQPHQWV
:HJHQHUDOO\UHFRPPHQGVHWWLQJXSDOO\RXU0\64/VHUYHUVDVGHVFULEHGLQ WKLVUHFLSHWRSUHYHQWDQ\SUREOHPV
*HWWLQJUHDG\
