tsukishima.ha at gmail
Dec 7, 2011, 2:22 AM
Post #7 of 7
(271 views)
Permalink
|
|
Re: $BN>%N!<%I$N(BDRBD$B$,(Bslave$B$N$^$^!"(Bprimary$B$K>:3J$7$^$;$s(B
[In reply to]
|
|
$BCfB<MM(B
$B$4O"Mm$"$j$,$H$&$4$6$$$^$9!#(B
> $B>e5-$N=hM}$G$O!"(BRA$B<+?H$,!V(Bnamenode$B!W$H$$$&%7%'%k$G$"$k$?$a!"(B
> $B<+J,<+?H$,(Bprep$B$G%R%C%H$7$F$$$^$7$?!#(B
$B$"!"$J$k$[.$I$G$9!#(B
$B5$$E$-$^$;$s$G$7$?!#(B
monitor_0$B$OF0:n$,$o$+$j$K$/$$$&$($K(B
$B%m%0%a%C%;!<%8$b$+$J$jIT?F@Z$J$N$G(B
$B<+:n(BRA$B$N>l9g$O!"4Y$j$d$9$$f+$G$9$M!D!#(B
Hadoop$BK\$OJY6/ITB-$N$?$aFI$s$G$$$J$$$N$G$9$,(B
$B$?$7$+(BACT/SBY$B$N(BDRBD$B9=@.$rNc$H$7$F2r@b$7$F$"$k$N$G$9$h$M!)(B
Master/Slave$B9=@.$G$bF1MM$N9=@.$,$H$l$k$H$$$&>pJs$r(B
$B$$$?$@$1$?$@$1$G$b$"$j$,$?$$$G$9!#(B
$B$^$?$J$K$+$"$l$P$4O"Mm$/$@$5$$!#(B
$B0J>e$h$m$7$/$*4j$$$$$?$7$^$9!#(B
$BCSED=_;R(B
2011$BG/(B12$B7n(B7$BF|(B17:50 $BCfB<!!7=B@(B <nakamura.keita [at] jp>:
> $BCSEDMM(B
>
> $B$*@$OC$K$J$C$F$*$j$^$9!#CfB<$G$9!#(B
>
> $B$4=u8@$"$j$,$H$&$4$6$$$^$7$?!#(B
> $B$^$?!"JV?.$,CY$/$J$j?=$7LuM-$j$^$;$s!#(B
> $B7kO@$H$7$F!"LdBj$,2r7h$7$^$7$?!#(B
>
>> hdpmaster38$B$G%/%i%9%?$r5/F0(B(service heartbeat start)$B$9$kA0$K(B
>> namenode_hadoop$B$H(Bjobtracker_hadoop$B$,$9$G$K5/F0$7$F$$$k(B
>> $B2DG=@-$O$"$j$^$;$s$+!#(B
>
> namenode_hadoop$B$H(Bjobtracker_hadoop$B$O5/F0$7$F$$$J$$$N$G$9$,!"(B
> RA$BFb$N(Bmonitor$B$K$*$$$F!"%W%m%;%93NG'$N=hM}$K8m$j$,$"$j$^$7$?!#(B
>
> monitor_namenode()
> {
> process=`/usr/bin/pgrep -f namenode`
> if [ -z "$process" ]; then
> $B!'(B
>
> $B>e5-$N=hM}$G$O!"(BRA$B<+?H$,!V(Bnamenode$B!W$H$$$&%7%'%k$G$"$k$?$a!"(B
> $B<+J,<+?H$,(Bprep$B$G%R%C%H$7$F$$$^$7$?!#(B
> $B7k2L!"(BHadoop$B$N(Bnamenode$B$,5/F0$7$F$$$J$/$F$b!"(B
> $B5/F0$7$F$$$k$H(Bmonitor$B$,H=CG$7$F$$$^$7$?!#(B
> $B!JCU@[.$JLdBj$K$D$$$F<ALd$7$F$7$^$$!"$9$_$^$;$s$G$7$?!#!K(B
>
> $BBP=h$O0J2<$N$h$&$K(Bps$B$r(Bjava$B$N%/%i%9L>$r%U%k%Q%9$G(Bgrep$B$9$k$h$&$K$7$^$7$?!#(B
>
> process=`ps -ef | grep org.apache.hadoop.mapred.JobTracker | grep -v grep`
>
> $B$4CzG+$KJV?.$7$F$$$?$@$-!"@?$K$"$j$,$H$&$4$6$$$^$7$?!#(B
> $B:#8e$H$b!"$I$&$>$h$m$7$/$*4j$$$$$?$7$^$9!#(B
>
> $B0J>e$G$9!#(B
>
> (2011/12/02 11:31), Junko IKEDA wrote:
>> $BCfB<MM(B
>>
>> hdpmaster38$B$G%/%i%9%?$r5/F0(B(service heartbeat start)$B$9$kA0$K(B
>> namenode_hadoop$B$H(Bjobtracker_hadoop$B$,$9$G$K5/F0$7$F$$$k(B
>> $B2DG=@-$O$"$j$^$;$s$+!#(B
>>
>> namenode RA$B$H(Bjobtracker RA$B$O(Bstart$B=hM}$N:G=i$K(B
>> monitor$B=hM}(B(monitor_jobtracker)$B$r8F$S=P$7$F$$$^$9!#(B
>> $B$3$l$O!"$9$G$K%j%=!<%9$,<B9T:Q$_$N>l9g$O(B
>> start$B=hM}$r>JN,$9$k$?$a$G$9!#(B
>>
>> ha-log_$B%N!<%I#2(B.txt$B$N(B160$B9TL\IU6a$r3NG'$9$k$H(B
>> namenode_hadoop$B$H(Bjobtracker_hadoop$B$N(B
>> monitor_0(start$B<B9T;~$K8F$S=P$5$l$k(Bmonitor$B=hM}(B)$B$,(B
>> rc=0(OCF_SUCCESS)$B$H$J$C$F$$$k$?$a!"$9$G$K%j%=!<%9$,5/F0$7$F$$$k(B
>> $B2DG=@-$,$"$j$^$9!#(B
>> $B5/F0$7$F$$$J$$>l9g$O!"8eB3$N(Bip_hadoop$B$d(Bfs_hadoop$B$N$h$&$K(B
>> rc=7(OCF_NOT_RUNNING)$B$,JV$j$^$9!#(B
>>
>> Dec 01 18:42:29 hdpmaster38 crmd: [30290]: info: process_lrm_event:
>> LRM operation namenode_hadoop_monitor_0 (call=4, rc=0, cib-update=8,
>> confirmed=true) ok
>> Dec 01 18:42:29 hdpmaster38 crmd: [30290]: info: process_lrm_event:
>> LRM operation jobtracker_hadoop_monitor_0 (call=5, rc=0, cib-update=9,
>> confirmed=true) ok
>> Dec 01 18:42:29 hdpmaster38 crmd: [30290]: info: process_lrm_event:
>> LRM operation ip_hadoop_monitor_0 (call=3, rc=7, cib-update=10,
>> confirmed=true) not running
>> Dec 01 18:42:29 hdpmaster38 crmd: [30290]: info: process_lrm_event:
>> LRM operation fs_hadoop_monitor_0 (call=2, rc=7, cib-update=11,
>> confirmed=true) not running
>>
>> $B0lEY!"N>%N!<%I$N%/%i%9%?$rDd;_$7$?$"$H(B
>> namenode_hadoop$B$H(Bjobtracker_hadoop$B$N>uBV$r3NG'$7$F$/$@$5$$!#(B
>> $B$^$?!"(BOS$B5/F0;~$K%W%m%;%9$,<+F05/F0$9$k$h$&$J(B
>> $B [at] _D$K$J$C$F$$$J$$$+!"$J$I$b3NG'$7$F$/$@$5$$!#(B
>>
>> $B$J$*!"(Bcrm$B$N [at] _D$G$9$,!"(Brsc_defaults$B$K(B
>> resource-stickiness$B$H(Bmigration-threshold$B$r(B
>> $B [at] _D$9$k$3$H$r$*4+$a$7$^$9!#(B
>>
>> $B;29M(B http://linux-ha.sourceforge.jp/wp/archives/1657/3
>>
>> resource-stickiness="INFINITY"$B$O<+F0%U%'%$%k%P%C%/$r%*%U$K$9$k [at] _D$G$9!#(B
>>
>> $B [at] _DjN(B
>> property \
>> no-quorum-policy="ignore" \
>> stonith-enabled="false" \
>> startup-fencing="false" \
>> crmd-transition-delay="2s"
>>
>> rsc_defaults \
>> resource-stickiness="INFINITY" \
>> migration-threshold="1"
>>
>>
>> $B0J>e$h$m$7$/$*4j$$$$$?$7$^$9!#(B
>>
>> $BCSED=_;R(B
>>
>>
>> 2011$BG/(B12$B7n(B1$BF|(B21:56 $BCfB<!!7=B@(B<nakamura.keita [at] jp>:
>>> $BCSEDMM(B
>>>
>>> $B$*@$OC$K$J$C$F$*$j$^$9!#CfB<$G$9!#(B
>>> $BB?$/$N$4=u8@$r$$$?$@$-!"$"$j$,$H$&$4$6$$$^$7$?!#(B
>>>
>>>>> location$B$N [at] _D$G$9$,!"(Bgroup$B$G$O$J$/(BMaster/Slave$B$N(B
>>>>> $B%9%3%"CM$r [at] _D$7$F$_$F$/$@$5$$!#(B
>>>>>
>>>>> location l_hadoop ms_drbd \
>>>>> rule role=master 200: #uname eq hdpmaster30 \
>>>>> rule role=master 100: #uname eq hdpmaster38
>>>
>>> $B>e5-$N [at] _D$K$h$j!"2r>C$G$-$^$7$?!#(B
>>> group$B$O(Bmaster$BB&$G5/F0$;$k$?$a!"(B
>>> ms$B$"$j$-$H$J$j!"(B
>>> location$B$O(Bms$B$r [at] _D$9$k$Y$-$J$N$G$9$M!#(B
>>> $BJY6/$K$J$j$^$9!#(B
>>>
>>> $BB3$1$FBgJQ62=L$G$9$,!"(B
>>> $B0J2<$K?7$7$$<ALd$r$5$;$F$/$@$5$$!#(B
>>>
>>> $B!cLdBj!d(B
>>> $BEvJ}$O!"(BHA$B%/%i%9%?>e$K(BHadoop$B$r9=C[$7$?$$$H9M$($F$*$j$^$9!#(B
>>> rg_hadoop$B$K!V(Bnamenode_hadoop$B!W$*$h$S!V(Bjobtracker_hadoop$B!W$H$$$&(B
>>> $B%j%=!<%9$rDI2C$7$?$N$G$9$,!"(B
>>> $B=LB`>uBV!JJR7O$N$_(Bheartbeat$B$,5/F0!K$+$i!"(B
>>> $BAj<j%N!<%I$N(Bheartbeat$B$r5/F0$5$;$k$H!"(B
>>> $B>e5-$N#2%j%=!<%9$,:F5/F0$7$F$7$^$$$^$9!#(B
>>>
>>> namenode_hadoop$B$*$h$S(Bjobtracker_hadoo$B$N5sF0$O0J2<$NN.$l$G$9!#(B
>>>
>>> $B=LB`>uBV!J%N!<%I#1$N(Bheartbeat$B$N$_5/F0!K$+$i%N!<%I#2$N(Bheartbeat$B$r5/F0(B
>>> $B"*%N!<%I#1$G$7$+>e5-$N#2%j%=!<%9$,5/F0$7$F$$$J$$$H;W$o$l$k$,!"(B
>>>$B!!!!(B $BN>%N!<%I$G5/F0$7$F$$$k$HG'<1$5$l$k!J%m%0#1!'(B166$B!"(B169$B9TL\!K(B
>>> $B"*%j%=!<%9$rDd;_!J%m%0#1!'(B222$B!"(B246$B9TL\!K!J%m%0#2!'(B179$B!"(B190$B9TL\!K(B
>>> $B"*%N!<%I#1$G%j%=!<%9$r5/F0!J%m%0#1!'(B260$B!"(B273$B9TL\!K(B
>>>
>>> $B%m%0#1!'%N!<%I#1$N%m%0!J85$+$i(Bheartbeat$B$,5/F0$7$F$$$?!K(B
>>> $B%m%0#2!'%N!<%I#2$N%m%0!J8e$+$i(Bheartbeat$B$r5/F0$7$?!K(B
>>>
>>> $B$J$*!"B>$N%j%=!<%9$O85$N%N!<%I$G5/F0$7$?$^$^@5>o$G$9!#(B
>>>
>>> $B=LB`>uBV$+$iDd;_$7$F$$$?%N!<%I$r%/%i%9%?$KI|5"$5$;$F$b!"(B
>>> $B#2%j%=!<%9$r85$N%N!<%I$G@5>o$K5/F0$7$?$^$^$H$9$k$K$O!"(B
>>> $B [at] _D$d(BRA$B$KLdBjE@$,$"$j$^$9$G$7$g$&$+!#(B
>>>
>>> $B!cHw9M!d(B
>>> $B!&>e5-$N#2%j%=!<%9$,5/F0$9$k%N!<%I$O!"(B
>>>$B!!(B Filesystem$B$d(BIPaddr2$B$N%j%=!<%9$,5/F0$7$F$$$k(B
>>>$B!!(B $B%N!<%I$G$"$kI,MW$,$"$j$^$9!#(B
>>>
>>> $B!&!V(BHadoop$BE0DlF~Lg!W$H$$$&=q@R$r;29M$K$7$F$*$j!"(B
>>>$B!!(B RA$B$O=q@R$N%5%s%W%k$r;HMQ$7$F$$$^$9!#(B
>>>
>>> $B!&%N!<%I#1$H%N!<%I#2$N5/F0=g$r5U$K$7$F$bF1MM$N7k2L$G$9!#(B
>>>
>>> $B!cE:IU!d(B
>>> $B!&(BHA-log$B!J%N!<%I#1!u%N!<%I#2!K(B
>>> $B!&(Bnamenode$B$N(BRA
>>> $B!&(Bjobtracker$B$N(BRA
>>> $B!&@_Dj%U%!%$%kN`(B
>>>
>>> $BBgJQ$*<j?t$rD:$-$^$7$F?=$7Lu$"$j$^$;$s$,!"(B
>>> $B$465<x$$$?$@$1$^$9$h$&!"$h$m$7$/$*4j$$$$$?$7$^$9!#(B
>>>
>>> $B0J>e$G$9!#(B
>>>
>>>
>>> (2011/12/01 11:02), Junko IKEDA wrote:
>>>> $B$9$_$^$;$s!#(B
>>>> $B0lIt%3%T%Z%_%9$G$9!#(B
>>>>
>>>> 3$B9TL\$N(Bmonitor$B [at] _D$O:o=|K:$l$G$9!#(B
>>>>
>>>>> primitive drbd_hadoop ocf:linbit:drbd \
>>>>> params drbd_resource="r_hadoop" \
>>>>> op monitor interval="10s" \ $B!z(B $B$3$N9T$OI,MW$J$7(B
>>>>> op start interval="0s" timeout="240s" on-fail="restart" \
>>>>> op monitor interval="10s" timeout="20s" on-fail="restart"
>>>>> role="Master" \
>>>>> op monitor interval="20s" timeout="20s" on-fail="restart" role="Slave" \
>>>>> op promote interval="0s" timeout="90s" on-fail="restart" \
>>>>> op demote interval="0s" timeout="90s" on-fail="block" \
>>>>> op stop interval="0s" timeout="100s" on-fail="block"
>>>>
>>>> $BCSED=_;R(B
>>>>
>>>> 2011$BG/(B12$B7n(B1$BF|(B10:56 Junko IKEDA<tsukishima.ha [at] gmail>:
>>>>> $BCfB<MM(B
>>>>>
>>>>> NTT$B%G!<%?@hC<5;=Q$NCSED$H?=$7$^$9!#(B
>>>>>
>>>>>> $B [at] _D%U%!%$%k$NN`$rE:IU$5$;$F$$$?$@$-$^$9!#(B
>>>>>> $B$J$*!"(BDRBD$B$N%j%=!<%9%(!<%8%'%s%H$O!"(Bdrbd-8.4.0$BF1:-$N$b$N$r;HMQ$7$F$$$^$9!#(B
>>>>>
>>>>> crm$B$N [at] _D$G$O(Bdrbd RA$B$N(Bprovider$B$,(Bheartbeat$B$H$J$C$F$$$^$9$,(B
>>>>> $B$3$l$O!"(BDRBD 8.4.0$B$N(BRA$B$r(B
>>>>> /usr/lib/ocf/resource.d/heartbeat/
>>>>> $B$K%3%T!<$5$l$?$H$$$&$3$H$G$7$g$&$+!#(B
>>>>> DRBD$B$N(BRA$B$O%G%U%)%k%H$G$O(B
>>>>> /usr/lib/ocf/resource.d/linbit/
>>>>> $B$K%$%s%9%H!<%k$5$l$^$9!#(B
>>>>>
>>>>> $B$h$C$F!"(Bdrbd RA$B$N [at] _D$G$ODL>o!"(Bprovider$B$K(Blinbit$B$r;XDj$7$^$9!#(B
>>>>> /usr/lib/ocf/resource.d/heartbeat/
>>>>> $B$K$b(Bdrbd RA$B$OB8:_$7$^$9$,!"$3$l$O(Bresource-agents$B$K4^$^$l$k8E$$(BRA$B$G$9!#(B
>>>>>
>>>>> $B$^$?!"(BMaster/Slave$B%j%=!<%9$N(Bmonitor$B [at] _D$G$9$,(B
>>>>> Master$B%j%=!<%9$H(BSlave$B%j%=!<%9$N(Binterval$BCM$r$=$l$>$l(B
>>>>> $BJL$NCM$H$7$F [at] _D$7$F$/$@$5$$!#(B
>>>>>
>>>>> drbd RA$B$N [at] _DjN(B
>>>>>
>>>>> primitive drbd_hadoop ocf:linbit:drbd \
>>>>> params drbd_resource="r_hadoop" \
>>>>> op monitor interval="10s" \
>>>>> op start interval="0s" timeout="240s" on-fail="restart" \
>>>>> op monitor interval="10s" timeout="20s" on-fail="restart"
>>>>> role="Master" \
>>>>> op monitor interval="20s" timeout="20s" on-fail="restart" role="Slave" \
>>>>> op promote interval="0s" timeout="90s" on-fail="restart" \
>>>>> op demote interval="0s" timeout="90s" on-fail="block" \
>>>>> op stop interval="0s" timeout="100s" on-fail="block"
>>>>>
>>>>> location$B$N [at] _D$G$9$,!"(Bgroup$B$G$O$J$/(BMaster/Slave$B$N(B
>>>>> $B%9%3%"CM$r [at] _D$7$F$_$F$/$@$5$$!#(B
>>>>>
>>>>> location l_hadoop ms_drbd \
>>>>> rule role=master 200: #uname eq hdpmaster30 \
>>>>> rule role=master 100: #uname eq hdpmaster38
>>>>>
>>>>> $B$^$?!"(BPacemaker 1.0.11$B$G$O!"(BMaster/Slave$B$NF0:n$G%P%0$,3NG'$5$l$F$$$^$9!#(B
>>>>> $B;29M!!(Bhttp://linux-ha.sourceforge.jp/wp/archives/2468/2
>>>>>
>>>>> $B>e5-$N%P%0$r2sHr$9$k$?$a$K!"(BMaster/Slave$B$@$1$G$O$J$/(B
>>>>> group$B$K4^$^$l$k3F%j%=!<%9$K$b(Blocation$B$r [at] _D$7$F$/$@$5$$!#(B
>>>>>
>>>>> location l_hadoop ms_drbd \
>>>>> rule role=master 200: #uname eq hdpmaster30 \
>>>>> rule role=master 100: #uname eq hdpmaster38 \
>>>>> rule role=master -inf: defined fail-count-fs_hadoop \
>>>>> rule role=master -inf: defined fail-count-ip_hadoop
>>>>>
>>>>> $B$J$*!">e5-$N%P%0$O(BPacemaker 1.0.12$B$G$O=$@5$5$l$F$$$^$9!#(B
>>>>>
>>>>> $B0J>e$h$m$7$/$*4j$$$$$?$7$^$9!#(B
>>>>>
>>>>> $BCSED=_;R(B
>>>>>
>>>>> 2011$BG/(B11$B7n(B30$BF|(B21:11 $BCfB<!!7=B@(B<nakamura.keita [at] jp>:
>>>>>> $BCfB<$H?=$7$^$9!#(B
>>>>>> $B$O$8$a$FEj9F$5$;$F$$$?$@$-$^$9!#(B
>>>>>> heartbeat$B$*$h$S(BDRBD$B$K4X$7$F!"0J2<$NLdBj$G:$$C$F$$$^$9!#(B
>>>>>>
>>>>>> $B!cLdBj!d(B
>>>>>> Heartbeat$B$rN>%N!<%I$G$[$\F1;~$K5/F0$5$;$k$H!"(BDRBD$B$,5/F0$O$9$k$N$G$9$,!"(B
>>>>>> $BN>%N!<%I6&$K(Bslave$B$N>uBV$,B3$-!"(Bprimary$B$K>:3J$7$^$;$s!#(B
>>>>>> $B$?$@$7!"JRJ}$N%N!<%I$,#H#e#a#r#t#b#e#a#t$K$*$1$k(BONLINE$B$N>uBV$^$G$J$C$?8e$K!"(B
>>>>>> $B0lJ}$N%N!<%I$rN)$A>e$2$k$H!"#D#R#B#D$,N>J}$H$b(Bslave$B$K$J$C$?8e!"(B
>>>>>> cib.xml$B$N(Blocation$B$G;XDj$7$?%N!<%I$,(Bprimary$B$K>:3J$7$^$9!#(B
>>>>>>
>>>>>> $B!c5?LdE@!d(B
>>>>>> $B!&(BHeartbeat$B$r5/F0$5$;$k%N!<%I$K$D$$$F$O!"(B
>>>>>>$B!!(B $B5/F0=gHV$r0U<1$9$kI,MW$,$"$k$N$G$7$g$&$+!)(B
>>>>>> $B!&EvJ}$N [at] _D$K!">e5-LdBj$N85$H$J$k2U=j$O$"$j$^$9$G$7$g$&$+!)(B
>>>>>>
>>>>>> $B!c4D6-!d(B
>>>>>> $B!&(Bpacemaker-1.0.11
>>>>>> $B!&(Bheartbeat-3.0.5
>>>>>> $B!&(Bdrbd-8.4.0
>>>>>>
>>>>>> $B [at] _D%U%!%$%k$NN`$rE:IU$5$;$F$$$?$@$-$^$9!#(B
>>>>>> $B$J$*!"(BDRBD$B$N%j%=!<%9%(!<%8%'%s%H$O!"(Bdrbd-8.4.0$BF1:-$N$b$N$r;HMQ$7$F$$$^$9!#(B
>>>>>>
>>>>>> ------- crm configure show -------
>>>>>> node $id="24606b2c-5a9d-4a86-9e83-a84ee941739d" hdpmaster38
>>>>>> node $id="57e51646-569b-4501-8073-61f06ec146bf" hdpmaster30
>>>>>> primitive drbd_hadoop ocf:heartbeat:drbd \
>>>>>> params drbd_resource="r_hadoop" \
>>>>>> op monitor interval="10s"
>>>>>> primitive fs_hadoop ocf:heartbeat:Filesystem \
>>>>>> params device="/dev/drbd0" directory="/hadoop/dfs/name" fstype="ext3"
>>>>>> options="noatime"
>>>>>> primitive ip_hadoop ocf:heartbeat:IPaddr2 \
>>>>>> params ip="10.171.208.87" nic="eth1" cidr_netmask="24" \
>>>>>> op monitor interval="10s"
>>>>>> group rg_hadoop fs_hadoop ip_hadoop
>>>>>> ms ms_drbd drbd_hadoop \
>>>>>> meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1"
>>>>>> notify="true"
>>>>>> location l_hadoop rg_hadoop 100: hdpmaster30
>>>>>> colocation c_hadoop inf: rg_hadoop ms_drbd:Master
>>>>>> order o_hadoop inf: ms_drbd:promote rg_hadoop:start
>>>>>> property $id="cib-bootstrap-options" \
>>>>>> dc-version="1.0.11-1554a83db0d3c3e546cfd3aaff6af1184f79ee87" \
>>>>>> cluster-infrastructure="Heartbeat" \
>>>>>> stonith-enabled="false" \
>>>>>> no-quorum-policy="ignore" \
>>>>>> default-resource-stickiness="200"
>>>>>>
>>>>>> $BBgJQ$*<j?t$G$9$,!"$I$J$?$+$465<x$/$@$5$$$^$9$h$&!"$*4j$$$$$?$7$^$9!#(B
>>>>>>
>>>>>> $B0J>e$G$9!#(B
>>>>>>
>>>>>> _______________________________________________
>>>>>> Linux-ha-japan mailing list
>>>>>> Linux-ha-japan [at] lists
>>>>>> http://lists.sourceforge.jp/mailman/listinfo/linux-ha-japan
>>>>>>
>>>>
>>>> _______________________________________________
>>>> Linux-ha-japan mailing list
>>>> Linux-ha-japan [at] lists
>>>> http://lists.sourceforge.jp/mailman/listinfo/linux-ha-japan
>>>>
>>>>
>>>
>>> _______________________________________________
>>> Linux-ha-japan mailing list
>>> Linux-ha-japan [at] lists
>>> http://lists.sourceforge.jp/mailman/listinfo/linux-ha-japan
>>>
>>
>> _______________________________________________
>> Linux-ha-japan mailing list
>> Linux-ha-japan [at] lists
>> http://lists.sourceforge.jp/mailman/listinfo/linux-ha-japan
>>
>>
>
> _______________________________________________
> Linux-ha-japan mailing list
> Linux-ha-japan [at] lists
> http://lists.sourceforge.jp/mailman/listinfo/linux-ha-japan
_______________________________________________
Linux-ha-japan mailing list
Linux-ha-japan [at] lists
http://lists.sourceforge.jp/mailman/listinfo/linux-ha-japan
|
$BE:IU(B.zip
