%!PS-Adobe-2.0 %%Creator: dvips by Radical Eye Software %%Title: atm_tr.dvi %%Pages: 20 1 %%BoundingBox: 0 0 612 792 %%EndComments %%BeginDocument: tex.pro /TeXDict 200 dict def TeXDict begin /bdf{bind def}def /bop-aux{}bdf /@rigin{ /@page-height exch def /@page-width exch def 72 Resolution div dup neg scale translate}bdf /@letter{Resolution dup -10 mul 8.5 11 @rigin}bdf /@landscape{[ 0 1 -1 0 0 0]concat Resolution dup 8.5 11 @rigin}bdf /@a4{Resolution dup -10.6929133858 mul 21 2.54 div 29.7 2.54 div @rigin}bdf /@legal{Resolution dup -13 mul 8.5 14 @rigin}bdf /@11x17{statusdict /11x17tray known{statusdict begin 11x17tray end}if Resolution dup -16 mul 11 17 @rigin}bdf /@manualfeed{ statusdict /manualfeed true put}bdf /@copies{/#copies exch def}bdf /@draft{ /bop-aux{gsave initmatrix 72 dup scale @page-width 2 div @page-height 2 div translate @page-height @page-width atan rotate /Helvetica-Bold findfont 2 scalefont setfont(DRAFT)dup stringwidth pop 2 div neg -1 moveto .95 setgray show grestore}bdf}bdf /@FontMatrix[1 0 0 -1 0 0]def /@FontBBox[0 0 0 0]def /dmystr(ZZf@@@)def /newname{dmystr cvn}bdf /df{/fontname exch def dmystr 2 fontname cvx(@@@@)cvs putinterval newname 7 dict def newname load begin /FontType 3 def /FontMatrix @FontMatrix def /FontBBox @FontBBox def /BitMaps 256 array def /BuildChar{CharBuilder}def /Encoding IdentityEncoding def end fontname{/foo setfont}2 array copy cvx def fontname load 0 dmystr 6 string copy cvn cvx put}bdf /dfe{newname dup load definefont setfont}bdf /ch-image{ ch-data 0 get}bdf /ch-width{ch-data 1 get}bdf /ch-height{ch-data 2 get}bdf /ch-xoff{ch-data 3 get}bdf /ch-yoff{ch-data 4 get}bdf /ch-dx{ch-data 5 get} bdf /CharBuilder{save 3 1 roll exch /BitMaps get exch get /ch-data exch def ch-data null ne{ch-dx 0 ch-xoff ch-yoff neg ch-xoff ch-width add ch-height ch-yoff sub setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-height ch-yoff sub .1 add]{ch-image}imagemask}if restore}bdf /dc{/ch-code exch def /ch-data exch def newname load /BitMaps get ch-code ch-data put}bdf /bop{gsave /SaveImage save def /bop-aux load exec 0 0 moveto}bdf /eop{clear SaveImage restore showpage grestore}bdf /@start{/Resolution exch def /IdentityEncoding 256 array def 0 1 255{IdentityEncoding exch 1 string dup 0 3 index put cvn put}for}bdf /p{show}bdf /RuleMatrix[1 0 0 -1 -.1 -.1]def /BlackDots 8 string def /v{gsave currentpoint translate false RuleMatrix{ BlackDots}imagemask grestore}bdf /a{moveto}bdf /delta 0 def /tail{dup /delta exch def 0 rmoveto}bdf /b{exch show tail}bdf /c{show delta 4 sub tail}bdf /d{ show delta 3 sub tail}bdf /e{show delta 2 sub tail}bdf /f{show delta 1 sub tail}bdf /g{show delta 0 rmoveto}bdf /h{show delta 1 add tail}bdf /i{show delta 2 add tail}bdf /j{show delta 3 add tail}bdf /k{show delta 4 add tail} bdf /l{show -4 0 rmoveto}bdf /m{show -3 0 rmoveto}bdf /n{show -2 0 rmoveto} bdf /o{show -1 0 rmoveto}bdf /q{show 1 0 rmoveto}bdf /r{show 2 0 rmoveto}bdf /s{show 3 0 rmoveto}bdf /t{show 4 0 rmoveto}bdf /w{0 rmoveto}bdf /x{0 exch rmoveto}bdf /y{3 2 roll show moveto}bdf /bos{/section save def}bdf /eos{clear section restore}bdf end %%EndDocument %%BeginDocument: special.pro TeXDict begin /SDict 200 dict def SDict begin /@SpecialDefaults{/hs 612 def /vs 792 def /ho 0 def /vo 0 def /hsc 1 def /vsc 1 def /ang 0 def /CLIP false def /BBcalc false def}bdf /@scaleunit 1 def /@hscale{@scaleunit div /hsc exch def}bdf /@vscale{@scaleunit div /vsc exch def}bdf /@hsize{/hs exch def /CLIP true def}bdf /@vsize{/vs exch def /CLIP true def}bdf /@hoffset{/ho exch def} bdf /@voffset{/vo exch def}bdf /@angle{/ang exch def}bdf /@rwi{10 div /rwi exch def}bdf /@llx{/llx exch def}bdf /@lly{/lly exch def}bdf /@urx{/urx exch def}bdf /@ury{/ury exch def /BBcalc true def}bdf end /@MacSetUp{userdict /md known{userdict /md get type /dicttype eq{md begin /letter{}def /note{}def /legal{}def /od{txpose 1 0 mtx defaultmatrix dtransform exch atan/pa exch def newpath clippath mark{transform{itransform moveto}}{transform{itransform lineto}}{6 -2 roll transform 6 -2 roll transform 6 -2 roll transform{ itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll curveto}}{{ closepath}}pathforall newpath counttomark array astore /gc xdf pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack}if}def /txpose{pxs pys scale ppr aload pop por{noflips{pop exch neg exch translate pop 1 -1 scale}if xflip yflip and{pop exch neg exch translate 180 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg translate}if xflip yflip not and{pop exch neg exch translate pop 180 rotate ppr 3 get ppr 1 get neg sub neg 0 translate}if yflip xflip not and{ppr 1 get neg ppr 0 get neg translate} if}{noflips{translate pop pop 270 rotate 1 -1 scale}if xflip yflip and{ translate pop pop 90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg translate}if xflip yflip not and{translate pop pop 90 rotate ppr 3 get ppr 1 get neg sub neg 0 translate}if yflip xflip not and{ translate pop pop 270 rotate ppr 2 get ppr 0 get neg sub neg 0 exch translate} if}ifelse scaleby96{ppr aload pop 4 -1 roll add 2 div 3 1 roll add 2 div 2 copy translate .96 dup scale neg exch neg exch translate}if}def /cp{pop pop showpage pm restore}def end}if}if}def /psf$TeXscale{65536 div}def /startTexFig {/psf$SavedState save def userdict maxlength dict begin Resolution 72 div dup neg scale currentpoint translate /psf$ury exch psf$TeXscale def /psf$urx exch psf$TeXscale def /psf$lly exch psf$TeXscale def /psf$llx exch psf$TeXscale def /psf$y exch psf$TeXscale def /psf$x exch psf$TeXscale def currentpoint /psf$cy exch def /psf$cx exch def /psf$sx psf$x psf$urx psf$llx sub div def /psf$sy psf$y psf$ury psf$lly sub div def psf$sx psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub translate /showpage{}def /erasepage{ }def /copypage{}def @MacSetUp}def /doclip{psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll exch lineto exch lineto exch lineto closepath clip newpath moveto}def /endTexFig{end psf$SavedState restore}def /@beginspecial{SDict begin /SpecialSave save def Resolution 72 div dup neg scale currentpoint translate @SpecialDefaults}bdf /@setspecial{CLIP{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto closepath clip}{initclip}ifelse ho vo translate hsc vsc scale ang rotate BBcalc{rwi urx llx sub div dup scale llx neg lly neg translate}if /showpage{}def newpath}bdf /@endspecial{clear SpecialSave restore end}bdf /@defspecial{SDict begin}bdf /@fedspecial{end}bdf /li{lineto}bdf /rl{rlineto} bdf /rc{rcurveto}bdf /np{/SaveX currentpoint /SaveY exch def def newpath}bdf /st{stroke SaveX SaveY moveto}bdf /fil{fill SaveX SaveY moveto}bdf /ellipse{ /endangle exch def /startangle exch def /yrad exch def /xrad exch def /savematrix matrix currentmatrix def translate xrad yrad scale 0 0 1 startangle endangle arc savematrix setmatrix}bdf end %%EndDocument TeXDict begin 300 @start /fa df[<071018F0307060706060C060C060C06080C080C480C4 C1C446C83870>14 14 2 0 19]97 dc[<383C1E0044C6630047028100460301008E0703000C06 03000C0603000C060600180C0600180C0620180C0C20180C0C403018044030180780>27 14 1 0 31]109 dc[<38F04518463846308C000C000C000C00180018001800180030003000>13 14 1 0 16]114 dc[<030003000600060006000600FFC00C000C000C0018001800180018003000308030 80310031001E00>10 20 1 0 13]116 dc[<0F1F0011A18020C38020C300418000018000018000 018000030000030200C30200E70400C5080078F000>17 14 1 0 20]120 dc dfe /fb df[<00 0001C00000078000001E00000078000001E00000078000000E00000038000000F0000003C00000 0F0000003C000000F0000000F00000003C0000000F00000003C0000000F0000000380000000E00 00000780000001E0000000780000001E0000000780000001C0>26 26 4 2 35]60 dc[<000100 030003000600060006000C000C000C00180018001800300030003000600060006000C000C000C0 0180018001800300030003000600060006000C000C000C00180018001800300030003000600060 006000C000C000C000>16 45 3 11 23]61 dc[<00FFFFE0000F0078000F003C000F001C000F00 1E001E001E001E001E001E001E001E001E003C003C003C003C003C0078003C00F0007803C0007F FF80007803C0007801E000F000F000F000F000F000F000F0007001E000F001E000F001E000F001 E000E003C001E003C003C003C0038003C00F0007801E00FFFFF000>31 31 2 0 34]66 dc[<00 01FC0000070700001C01C0003000E000E000E001C000700380007007800078070000380F000038 1E0000381E0000383C0000383C00007878000078780000787800007878000078F00000F0F00000 F0F00000E0F00001E0F00001C0F00003C070000380701C070070600F0038811E0038813C001C81 70000E81E0000783808001FD0080000101800001010000038300000386000003FE000003FC0000 01F8000000F000>29 41 2 9 36]81 dc[<00FFFF80000F01E0000F0070000F0038000F003C00 1E003C001E003C001E003C001E003C003C0078003C0078003C00F0003C01E00078038000780F00 007FF80000781C0000F00E0000F00F0000F0070000F0078001E00F0001E00F0001E00F0001E00F 0003C01E0003C01E0203C01E0203C01E0407C00E04FFFC0718000003E0>31 32 2 1 35]82 dc[ <0007E0800018118000300B000060070000C0070001C003000180020003800200038002000380 0200038000000380000003C0000003F8000003FF800001FFC00000FFE000003FF0000003F00000 00F0000000700000007000000070002000700020007000200060006000E0006000C0006001C000 70018000E8030000C60E000081F80000>25 33 3 1 28]83 dc[<0FFFFFFC1E03C0381803C018 1003C0082003C00820078008600780084007800840078008800F0010000F0000000F0000000F00 00001E0000001E0000001E0000001E0000003C0000003C0000003C0000003C0000007800000078 0000007800000078000000F0000000F0000000F0000000F0000001F000007FFFC000>30 31 1 0 27]84 dc[45 32 2 1 43]87 dc[<0000780003F80000700000700000700000700000E00000E00000 E00000E00001C00001C000F1C00389C00707800E03801C03803C03803807007807007807007807 00F00E00F00E00F00E00F00E10F01C20F01C20703C20705C40308C400F0780>21 32 2 0 24] 100 dc[<007C018203010603060706060E00078007F803FC01FE001F00077007F006F006E00440 0820301FC0>16 20 2 0 21]115 dc dfe /fc df[<0C003C00CC000C000C000C000C000C000C 000C000C000C000C000C000C00FF80>9 16 2 0 15]49 dc[<1F00618040C08060C06000600060 00C00180030006000C00102020207FC0FFC0>11 16 1 0 15]50 dc[<1F00218060C060C000C0 008001800F00008000400060C060C060804060801F00>11 16 1 0 15]51 dc[<030003000700 0F000B001300330023004300C300FFE003000300030003001FE0>11 16 1 0 15]52 dc[<2080 3F002C002000200020002F0030802040006000600060C06080C061801F00>11 16 1 0 15]53 dc dfe /fd df[<03000700FF0007000700070007000700070007000700070007000700070007 0007000700070007007FF0>12 21 2 0 18]49 dc[<0F8030E040708030C038E0384038003800 700070006000C00180030006000C08080810183FF07FF0FFF0>13 21 2 0 18]50 dc[<0FE030 306018701C701C001C00180038006007E000300018000C000E000EE00EE00EC00C401830300FE0 >15 21 1 0 18]51 dc[<00300030007000F001F001700270047008701870107020704070C070 FFFE0070007000700070007003FE>15 21 1 0 18]52 dc[<20303FE03FC02400200020002000 20002F8030E020700030003800384038E038E0388030406020C01F00>13 21 2 0 18]53 dc dfe /fe df[20 2 3 -7 27]0 dc dfe /ff df[26 2 4 -10 35]0 dc[<400004C0000C6000183000301800600C00C006018003030001860000CC000078 0000300000300000780000CC000186000303000601800C00C0180060300030600018C0000C4000 04>22 24 6 1 35]2 dc[<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFFFFFFFFFF7FFE7FFE 3FFC1FF80FF003C0>16 18 3 -2 23]15 dc[26 36 4 7 35]21 dc dfe /fg df[<00003F03E00000C386700001878CF000 03879CF00003031860000700380000070038000007003800000E003800000E007000000E007000 000E00700000FFFFFF80001C007000001C00E000001C00E000001C00E000001C00E000003800E0 00003801C000003801C000003801C000003801C000007001C00000700380000070038000007003 80000070038000006003800000E007000000E007000000E007000000E007000000C006000001C0 0E000001C00E000031860C0000798F180000F31E100000620C6000003C07C00000>36 41 -2 9 28]11 dc[<00003FE00000E0100001803800038078000300780007003000070000000700000007 0000000E0000000E0000000E000000FFFFE0000E00E0001C01C0001C01C0001C01C0001C01C000 1C03800038038000380380003803800038070000380700007007000070071000700E2000700E20 00700E2000E00E2000E0064000E0038000E0000000C0000001C0000001C0000031800000798000 00F3000000620000003C000000>29 41 -2 9 26]12 dc[<00003FCE0000E0EE000181FC000381 FC000300DC0007001C0007003800070038000E0038000E0038000E0070000E007000FFFFF0001E 0070001C00E0001C00E0001C00E0001C00E0001C01C0003801C0003801C0003801C00038038000 380380007003800070038800700710007007100070071000E0071000E0032000E001C000E00000 00C0000001C0000001C000003180000079800000F3000000620000003C000000>31 41 -2 9 27 ]13 dc[<0E0E1F1F3F3F3F3F1D1D01010202020204040404080810102020C0C0>16 14 7 -18 23]34 dc[<000F00000030800C0060401E00C0203C01C0201C0180E0040381E0080303C0080701 80100700002007000040060001800678028007C40C4007041020060420200F0440101D0840103C F0581038007C10780078107800301078000010F0000020F0000020F0000040F000004070000080 7000010038000300180004000E0018000780E00000FF0000>31 34 5 1 35]38 dc[<7FF0FFE0 7FE0>12 3 3 -8 16]45 dc[<000F800030E000E07001C0700380300380380700380F00780F00 780E00781E00781E00703C00F03C00F03C00F03C00F07801E07801E07801E07801C07003C0F003 C0F00380F00780F00700700700700E00701C003038001870000FC000>21 31 4 1 23]48 dc[< 000200020006000E003C00DC031C001C0038003800380038007000700070007000E000E000E000 E001C001C001C001C003800380038003800780FFF8>15 30 5 0 23]49 dc[<001F0000618000 80E00100E00200700220700420700410700820F00820F00820F00840E00881E00703C000038000 0700000C000018000060000080000300000400000800401000401000802001807E030047FF0041 FE0080FC00807800>20 31 4 1 23]50 dc[<001F800060E00080700100300200380420380420 380410380420700460700380600000E00001C000030000FE00001C000006000007000007800007 80000780300780780780780780F00F00800F00401E00401C0040380020E0001F8000>21 31 4 1 23]51 dc[<0007C0001C200030200060E000C1E00181E00380C00700000F00000E00001E00001E 78001D84003E06003E07003C07007C0780780780780780780780700F00700F00F00F00F00E00F0 1E00701C00601C0070380030700010C0000F8000>19 31 5 1 23]54 dc[<08E0100BF01017F0 201FF0603E19C0380E80200080600100400300800300000600000E00000C00001C00001C000038 0000380000700000700000F00000F00001E00001E00001E00003C00003C00003C00007C0000780 00078000030000>20 31 7 1 23]55 dc[<001F000061800080C0010060030060060060060060 0600600E00C00F00800F818007C30007E40003F80001F80003FC00047E00183F00300F00200700 600700C00300C00300C00300800600800600C00C00C008004030003060001F8000>19 31 5 1 23]56 dc[<001F0000718000C0C00180C00380E00700E00F00E00F01E01E01E01E01E01E01E01E 01C01C03C01C03C01C03C01C07C01C0F800C0F8006378003C700000F00000E00000E00001C0060 1C00F03800F07000E0600080C0004380003E0000>19 31 5 1 23]57 dc[<0000020000000600 0000060000000E0000001E0000001E0000003F0000002F0000004F0000004F0000008F0000010F 0000010F0000020F0000020F0000040F00000C0F0000080F0000100F0000100F0000200F80003F FF800040078000C007800080078001000780010007800200078002000780060007801E000F80FF 807FF8>29 32 2 0 34]65 dc[<0000FE0200078186001C004C0038003C0060003C00C0001C01 C0001803800018070000180F0000181E0000101E0000103C0000003C0000007800000078000000 7800000078000000F0000000F0000000F0000000F0000000F00000807000008070000080700001 003800010038000200180004000C001800060020000381C00000FE0000>31 33 6 1 33]67 dc[ <01FFFF80001E00E0001E0070001E0038001E001C003C001C003C000E003C000E003C000E0078 000E0078000E0078000E0078000E00F0001E00F0001E00F0001E00F0001E01E0003C01E0003C01 E0003C01E0007803C0007003C0007003C000E003C001C0078001C00780038007800E0007801C00 0F007000FFFFC000>31 31 3 0 34]68 dc[<01FFFFFE001E001C001E000C001E0004001E0004 003C0004003C0004003C0004003C00040078080800780800007808000078180000F0300000FFF0 0000F0300000F0300001E0200001E0200001E0200001E0001003C0002003C0002003C0004003C0 0040078000800780018007800100078007000F001F00FFFFFE00>31 31 3 0 31]69 dc[<0000 FC040007030C001C00980030007800E0007801C000380380003003800030070000300E0000301E 0000201E0000203C0000003C00000078000000780000007800000078000000F0000000F000FFF0 F0000780F0000780F0000F0070000F0070000F0070000F0070001E0038001E0018003E001C002E 000E00CC000383040000FC0000>30 33 6 1 35]71 dc[<01FFF0001F00001E00001E00001E00 003C00003C00003C00003C0000780000780000780000780000F00000F00000F00000F00001E000 01E00001E00001E00003C00003C00003C00003C0000780000780000780000780000F8000FFF800 >20 31 3 0 18]73 dc[<01FFF800001F0000001E0000001E0000001E0000003C0000003C0000 003C0000003C00000078000000780000007800000078000000F0000000F0000000F0000000F000 0001E0000001E0000001E0000001E0008003C0010003C0010003C0030003C00200078006000780 060007800C0007801C000F007800FFFFF800>25 31 3 0 29]76 dc[<01FE00007FC0001E0000 FC00001E0000F80000170001780000170001780000270002F00000270004F00000270004F00000 270008F00000470009E00000470011E00000470021E00000470021E00000870043C00000838043 C00000838083C00000838083C00001038107800001038207800001038207800001038407800002 03840F00000203880F00000203900F00000203900F00000401E01E00000401E01E00000401C01E 00000C01801E00001C01803E0000FF8103FFC000>42 31 3 0 41]77 dc[<01FF007FE0001F00 0F00001F0004000017800400001780040000278008000023C008000023C008000023C008000041 E010000041E010000041F010000040F010000080F0200000807820000080782000008078200001 003C400001003C400001003C400001001E400002001E800002001E800002000F800002000F8000 04000F0000040007000004000700000C000700001C00020000FF80020000>35 31 3 0 34]78 dc[<0001FC0000070700001C01C0003000E000E0006001C000700380007007800038070000380E 0000381E0000381C0000383C0000383C00003878000078780000787800007878000078F00000F0 F00000F0F00000E0F00001E0F00001C0F00003C0700003807000070078000F0038001E0038003C 001C0070000E00E0000783800001FC0000>29 33 6 1 35]79 dc[<01FFFF80001E00E0001E00 70001E0038001E003C003C003C003C003C003C003C003C003C0078007800780078007800F00078 00E000F003C000F00F0000FFFC0000F0000001E0000001E0000001E0000001E0000003C0000003 C0000003C0000003C00000078000000780000007800000078000000F800000FFF00000>30 31 3 0 31]80 dc[<01FFFF00001E03C0001E00E0001E0070001E0078003C0078003C0078003C007800 3C0078007800F0007800F0007801E0007801C000F0070000F01E0000FFF00000F0380001E01C00 01E01E0001E00E0001E00F0003C01E0003C01E0003C01E0003C01E0007803C0007803C0807803C 0807803C100F801C10FFF00C20000007C0>29 32 3 1 33]82 dc[<0007E040001C18C0003005 800060038000C0038001C00180018001000380010003800100038001000380000003C0000003C0 000003F8000001FF800001FFE000007FF000001FF0000001F80000007800000078000000380000 00380020003800200038002000300060007000600060006000E0007000C000E8038000C6060000 81F80000>26 33 3 1 26]83 dc[<0FFFFFF01E0780E0180780201007802020078020200F0020 600F0020400F0020400F0020801E0040001E0000001E0000001E0000003C0000003C0000003C00 00003C00000078000000780000007800000078000000F0000000F0000000F0000000F0000001E0 000001E0000001E0000001E0000003E00000FFFF0000>28 31 8 0 33]84 dc[<7FFC1FF807C0 03C00780010007800100078001000F0002000F0002000F0002000F0002001E0004001E0004001E 0004001E0004003C0008003C0008003C0008003C00080078001000780010007800100078001000 F0002000F0002000F0002000F0004000F0004000700080007001000030020000380400000C1800 0007E00000>29 32 9 1 34]85 dc[41 32 9 1 45]87 dc[<00FFF07FE0000F801F00000F001C00 000F801000000780300000078020000007C040000003C080000003C100000003E200000001E400 000001EC00000001F800000000F000000000F800000000F800000000F8000000017C000000023C 000000063C000000043E000000081E000000101E000000201F000000400F000000800F00000180 0F8000010007800007000780001F000FC000FFC07FF800>35 31 2 0 34]88 dc[<00F1800389 C00707800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E20F01C 40F01C40703C40705C40308C800F0700>19 20 4 0 23]97 dc[<07803F800700070007000700 0E000E000E000E001C001C001CF01D0C3A0E3C0E380F380F700F700F700F700FE01EE01EE01EE0 1CE03CE038607060E031C01F00>16 32 5 0 21]98 dc[<007E0001C1000300800E07801E0780 1C07003C0200780000780000780000F00000F00000F00000F00000F00000700100700200300400 18380007C000>17 20 4 0 21]99 dc[<0000780003F80000700000700000700000700000E000 00E00000E00000E00001C00001C000F1C00389C00707800E03801C03803C038038070078070078 0700780700F00E00F00E00F00E00F00E20F01C40F01C40703C40705C40308C800F0700>21 32 4 0 23]100 dc[<007C01C207010E011C013C013802780C7BF07C00F000F000F000F00070007001 70023804183807C0>16 20 4 0 21]101 dc[<00007800019C00033C00033C0007180007000007 00000E00000E00000E00000E00000E0001FFE0001C00001C00001C00001C000038000038000038 0000380000380000700000700000700000700000700000700000E00000E00000E00000E00000C0 0001C00001C0000180003180007B0000F300006600003C0000>22 41 -2 9 14]102 dc[<003C 6000E27001C1E00380E00700E00F00E00E01C01E01C01E01C01E01C03C03803C03803C03803C03 803C07003C07001C0F001C17000C2E0003CE00000E00000E00001C00001C00301C00783800F070 0060E0003F8000>20 29 2 9 21]103 dc[<01E0000FE00001C00001C00001C00001C000038000 038000038000038000070000070000071E000763000E81800F01C00E01C00E01C01C03801C0380 1C03801C0380380700380700380700380E10700E20700C20701C20700C40E00CC0600700>20 32 3 0 23]104 dc[<00C001E001E001C000000000000000000000000000000E0033002300438043 00470087000E000E000E001C001C001C003840388030807080310033001C00>11 31 4 0 14] 105 dc[<01E0000FE00001C00001C00001C00001C0000380000380000380000380000700000700 000703C00704200E08E00E11E00E21E00E40C01C80001D00001E00001FC00038E0003870003870 00383840707080707080707080703100E03100601E00>19 32 3 0 21]107 dc[<03C01FC00380 03800380038007000700070007000E000E000E000E001C001C001C001C00380038003800380070 00700070007100E200E200E200E20064003800>10 32 4 0 12]108 dc[<1C0F80F0002630C318 004740640C004780680E004700700E004700700E008E00E01C000E00E01C000E00E01C000E00E0 1C001C01C038001C01C038001C01C038001C01C0708038038071003803806100380380E1003803 80620070070066003003003800>33 20 4 0 37]109 dc[<1C0F802630C0474060478060470070 4700708E00E00E00E00E00E00E00E01C01C01C01C01C01C01C0384380388380308380708380310 7003303001C0>22 20 4 0 26]110 dc[<007C0001C3000301800E01C01E01C01C01E03C01E078 01E07801E07801E0F003C0F003C0F003C0F00780F00700700F00700E0030180018700007C000> 19 20 4 0 23]111 dc[<01C1E002621804741C04781C04701E04701E08E01E00E01E00E01E00 E01E01C03C01C03C01C03C01C0380380780380700380E003C1C0072380071E000700000700000E 00000E00000E00000E00001C00001C0000FFC000>23 29 0 9 23]112 dc[<00F0400388C00705 800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E00F01C00F01C 00703C00705C0030B8000F380000380000380000700000700000700000700000E00000E0000FFE 00>18 29 4 9 21]113 dc[<1C1E002661004783804787804707804703008E00000E00000E0000 0E00001C00001C00001C00001C0000380000380000380000380000700000300000>17 20 4 0 19]114 dc[<00FC030206010C030C070C060C000F800FF007F803FC003E000E700EF00CF00CE0 08401020601F80>16 20 3 0 19]115 dc[<018001C0038003800380038007000700FFF007000E 000E000E000E001C001C001C001C003800380038003820704070407080708031001E00>12 28 4 0 15]116 dc[<0E00C03300E02301C04381C04301C04701C08703800E03800E03800E03801C07 001C07001C07001C07101C0E20180E20180E201C1E200C264007C380>20 20 4 0 24]117 dc[< 0E03803307802307C04383C04301C04700C08700800E00800E00800E00801C01001C01001C0100 1C02001C02001C04001C04001C08000E300003C000>18 20 4 0 21]118 dc[<0E00C1C03300E3 C02301C3E04381C1E04301C0E04701C060870380400E0380400E0380400E0380401C0700801C07 00801C0700801C0701001C0701001C0602001C0F02000C0F04000E13080003E1F000>27 20 4 0 30]119 dc[<0383800CC4401068E01071E02071E02070C040E00000E00000E00000E00001C000 01C00001C00001C040638080F38080F38100E5810084C600787800>19 20 3 0 21]120 dc[<0E 00C03300E02301C04381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C 07001C0E00180E00180E001C1E000C3C0007DC00001C00001C00003800F03800F07000E06000C0 C0004380003E0000>19 29 4 9 22]121 dc[<01C04003E08007F1800C1F000802000004000008 000010000020000040000080000100000200000401000802001002003E0C0063FC0041F80080E0 00>18 20 3 0 19]122 dc dfe /fh df[<000FF07F00007FF9FF8000F83FC7C001E07F8FC003 E07F0FC007C07F0FC007C03F078007C01F000007C01F000007C01F000007C01F000007C01F0000 FFFFFFF800FFFFFFF80007C01F000007C01F000007C01F000007C01F000007C01F000007C01F00 0007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F 000007C01F000007C01F00003FF8FFF0003FF8FFF000>34 32 0 0 31]11 dc[12 4 1 -8 17]45 dc[<387CFEFEFE7C38>7 7 4 0 15]46 dc[<00E00001E0000FE000FF E000F3E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003 E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E000FFFF80FFFF80> 17 29 4 0 26]49 dc[<07F0001FFE00383F007C1F80FE0FC0FE0FC0FE0FE0FE07E07C07E03807 E0000FE0000FC0000FC0001F80001F00003E0000780000F00000E00001C0000380600700600E00 601C00E01FFFC03FFFC07FFFC0FFFFC0FFFFC0>19 29 3 0 26]50 dc[<01FC0007FF000E0F80 1E0FC03F07E03F07E03F07E03F07E01E0FC0000FC0000F80001F0001FC0001FC00000F800007C0 0003E00003F00003F83803F87C03F8FE03F8FE03F8FE03F0FC03F07807E03C0FC01FFF8003FC00 >21 29 2 0 26]51 dc[<0001C00003C00007C00007C0000FC0001FC0003BC00073C00063C000 C3C00183C00383C00703C00E03C00C03C01803C03803C07003C0E003C0FFFFFEFFFFFE0007C000 07C00007C00007C00007C00007C000FFFE00FFFE>23 29 1 0 26]52 dc[<3803803FFF803FFF 003FFE003FFC003FF0003F800030000030000030000030000033F80037FE003C1F00380F801007 C00007C00007E00007E07807E0FC07E0FC07E0FC07E0FC07C0780FC0600F80381F001FFC0007F0 00>19 29 3 0 26]53 dc[<003F0001FFC007E0E00F81E01F03F01E03F03E03F07C03F07C01E0 7C0000FC1000FCFF00FDFFC0FD03E0FE01F0FE01F0FC01F8FC01F8FC01F8FC01F87C01F87C01F8 7C01F83C01F03E01F01E03E00F07C007FF8001FE00>21 29 2 0 26]54 dc[<6000007FFFF87F FFF87FFFF07FFFE07FFFC0E00180C00300C00300C00600000C0000180000380000380000780000 700000F00000F00001F00001F00001F00001F00003F00003F00003F00003F00003F00003F00001 E00000C000>21 30 3 0 26]55 dc[<387CFEFEFE7C38000000000000387CFEFEFE7C38>7 20 4 0 15]58 dc[<0000E000000000E000000001F000000001F000000001F000000003F800000003F8 00000006FC00000006FC0000000EFE0000000C7E0000000C7E000000183F000000183F00000030 3F800000301F800000701FC00000600FC00000600FC00000C007E00000FFFFE00001FFFFF00001 8003F000018003F000030001F800030001F800060001FC00060000FC000E0000FE00FFE00FFFE0 FFE00FFFE0>35 31 2 0 40]65 dc[31 31 2 0 37]66 dc[<0007FC0200 3FFF0E00FE03DE03F000FE07E0003E0FC0001E1F80001E3F00000E3F00000E7F0000067E000006 7E000006FE000000FE000000FE000000FE000000FE000000FE000000FE0000007E0000007E0000 067F0000063F0000063F00000C1F80000C0FC0001807E0003803F0007000FE01C0003FFF800007 FC00>31 31 3 0 38]67 dc[30 31 2 0 34]69 dc[ 28 31 2 0 33]70 dc[16 31 2 0 20]73 dc[36 31 2 0 41]75 dc[26 31 2 0 31]76 dc[45 31 2 0 50]77 dc[30 31 2 0 36]80 dc[36 31 2 0 39]82 dc[<03FC080FFF381E03F83800F8700078 700038F00038F00018F00018F80000FC00007FC0007FFE003FFF801FFFE00FFFF007FFF000FFF8 0007F80000FC00007C00003CC0003CC0003CC0003CE00038E00078F80070FE01E0E7FFC081FF00 >22 31 3 0 29]83 dc[<7FFFFFFC7FFFFFFC7C07E07C7007E01C6007E00C6007E00CE007E00E C007E006C007E006C007E006C007E0060007E0000007E0000007E0000007E0000007E0000007E0 000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007 E0000007E00003FFFFC003FFFFC0>31 30 2 0 36]84 dc[<07FC001FFF003F0F803F07C03F03 E03F03E00C03E00003E0007FE007FBE01F03E03C03E07C03E0F803E0F803E0F803E0FC05E07E0D E03FF8FE0FE07E>23 20 1 0 25]97 dc[24 32 2 0 29]98 dc[<01FE0007FF801F0FC03E0FC03E0FC07C0FC07C0300FC0000FC0000FC0000FC00 00FC0000FC00007C00007E00003E00603F00C01F81C007FF0001FC00>19 20 2 0 23]99 dc[< 0007F80007F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F801F8F8 0FFEF81F83F83E01F87E00F87C00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00F8FC00F87C00F8 7C00F87E00F83E01F81F07F80FFEFF03F8FF>24 32 2 0 29]100 dc[<01FE0007FF800F83C01E 01E03E00F07C00F07C00F8FC00F8FFFFF8FFFFF8FC0000FC0000FC00007C00007C00003E00181E 00180F807007FFE000FF80>21 20 1 0 24]101 dc[<001F8000FFC001F3E003E7E003C7E007C7 E007C3C007C00007C00007C00007C00007C000FFFC00FFFC0007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0003FFC003FFC 00>19 32 1 0 16]102 dc[<01FC3C07FFFE0F079E1E03DE3E03E03E03E03E03E03E03E03E03E0 1E03C00F07800FFF0009FC001800001800001C00001FFF800FFFF007FFF81FFFFC3C007C70003E F0001EF0001EF0001E78003C78003C3F01F80FFFE001FF00>23 30 1 10 26]103 dc[24 32 3 0 29]104 dc[<1C003E003F007F003F003E001C 00000000000000000000000000FF00FF001F001F001F001F001F001F001F001F001F001F001F00 1F001F001F001F001F00FFE0FFE0>11 33 2 0 14]105 dc[24 32 2 0 28]107 dc[11 32 2 0 14]108 dc[39 20 3 0 44]109 dc[24 20 3 0 29]110 dc[<01 FF0007FFC01F83F03E00F83E00F87C007C7C007CFC007EFC007EFC007EFC007EFC007EFC007E7C 007C7C007C3E00F83E00F81F83F007FFC001FF00>23 20 1 0 26]111 dc[24 29 2 9 29]112 dc[18 20 2 0 22]114 dc[<0FE63FFE701E600EE006E006F800FFC07FF83FFC1FFE03FE001FC007C007E007F0 06F81EFFFCC7F0>16 20 2 0 21]115 dc[<01800180018003800380038007800F803F80FFFCFF FC0F800F800F800F800F800F800F800F800F800F800F860F860F860F860F8607CC03F801F0>15 29 1 0 20]116 dc[24 20 3 0 29]117 dc[25 20 1 0 28]118 dc[35 20 1 0 38]119 dc[25 20 1 0 28]120 dc[25 29 1 9 28]121 dc dfe /fi df[<00 FC000182000703000607000E02000E00000E00000E00000E00000E0000FFFF000E07000E07000E 07000E07000E07000E07000E07000E07000E07000E07000E07000E07000E07000E07007F0FE0> 19 26 0 0 21]12 dc[<007E1F8001C170400703C060060380E00E0380400E0380000E0380000E 0380000E0380000E038000FFFFFFE00E0380E00E0380E00E0380E00E0380E00E0380E00E0380E0 0E0380E00E0380E00E0380E00E0380E00E0380E00E0380E00E0380E00E0380E07F8FE3FC>30 26 0 0 32]14 dc[<60C0F1E0F9F068D0081008100810102010202040C180>12 11 1 -15 19]34 dc[<60F0F07010101020204080>4 11 3 7 11]44 dc[10 2 0 -7 13]45 dc[<60 F0F060>4 4 3 0 11]46 dc[<0004000C00180018001800300030003000600060006000C000C0 00C00180018001800300030003000600060006000C000C000C0018001800180030003000300060 0060006000C000C000>14 37 2 9 19]47 dc[<078018603030303060186018E01CE01CE01CE0 1CE01CE01CE01CE01CE01CE01CE01CE01C601860187038303018600780>14 24 2 0 19]48 dc[ <03000700FF000700070007000700070007000700070007000700070007000700070007000700 0700070007000700FFF0>12 24 3 0 19]49 dc[<0F80106020304038803CC01CE01C401C003C 003800380070006000C001800100020004040804100430083FF87FF8FFF8>14 24 2 0 19]50 dc[<0F8010E02070607870382038007800700070006000C00F8000E000700038003C003CE03CE0 3CC03C4038407030E00F80>14 24 2 0 19]51 dc[<00300030007000F000F001700370027004 700C7008701070307020704070C070FFFF00700070007000700070007007FF>16 24 1 0 19] 52 dc[<30183FF03FE03FC02000200020002000200027C03860203000380018001C001C401CE0 1CE01C80184038403030E00F80>14 24 2 0 19]53 dc[<40007FFE7FFC7FFC40088010801080 2000400040008001800180010003000300030003000700070007000700070007000200>15 25 2 0 19]55 dc[<07801860303070306018E018E018E01CE01CE01C601C603C303C185C0F9C001C00 180018003870307060604021801F00>14 24 2 0 19]57 dc[<60F0F060000000000000000060 F0F07010101020204080>4 23 3 7 11]59 dc[<000C0000000C0000000C0000001E0000001E00 00003F000000270000002700000043800000438000004380000081C0000081C0000081C0000100 E0000100E00001FFE000020070000200700006007800040038000400380008001C0008001C001C 001E00FF00FFC0>26 26 1 0 29]65 dc[22 26 2 0 27]66 dc[<003F0201C0C603002E0E 001E1C000E1C0006380006780002700002700002F00000F00000F00000F00000F00000F0000070 00027000027800023800041C00041C00080E000803003001C0C0003F00>23 26 2 0 28]67 dc[ 24 26 2 0 29]68 dc[20 26 2 0 25]70 dc[11 26 1 0 14]73 dc[19 26 2 0 24]76 dc[30 26 2 0 35]77 dc[24 26 2 0 29]78 dc[<007F000001C1C00007007000 0E0038001C001C003C001E0038000E0078000F0070000700F0000780F0000780F0000780F00007 80F0000780F0000780F0000780F000078078000F0078000F0038000E003C001E001C001C000E00 38000700700001C1C000007F0000>25 26 2 0 30]79 dc[21 26 2 0 26]80 dc[25 26 2 0 28]82 dc[<0FC2183620 0E6006C006C002C002C002E00070007E003FE01FF807FC003E000E00070003800380038003C002 C006E004D81887E0>16 26 2 0 21]83 dc[<7FFFFF00701C0700401C0100401C0100C01C0180 801C0080801C0080801C0080001C0000001C0000001C0000001C0000001C0000001C0000001C00 00001C0000001C0000001C0000001C0000001C0000001C0000001C0000001C0000001C0000001C 000003FFE000>25 26 1 0 28]84 dc[24 26 2 0 29]85 dc[36 26 1 0 39]87 dc[7 37 3 9 11]91 dc[<1830204040804080810081008100B160F9F0 78F03060>12 11 5 -15 19]92 dc[7 37 0 9 11]93 dc[<3F8070C070E020700070007007F01C 7030707070E070E071E071E0F171FB1E3C>16 16 2 0 19]97 dc[18 26 1 0 21]98 dc[< 07F80C1C381C30087000E000E000E000E000E000E0007000300438080C1807E0>14 16 1 0 17] 99 dc[<007E00000E00000E00000E00000E00000E00000E00000E00000E00000E0003CE000C3E 00380E00300E00700E00E00E00E00E00E00E00E00E00E00E00E00E00600E00700E00381E001C2E 0007CFC0>18 26 1 0 21]100 dc[<07C01C3030187018600CE00CFFFCE000E000E000E0006000 300438080C1807E0>14 16 1 0 17]101 dc[<01F0031807380E100E000E000E000E000E000E00 FFC00E000E000E000E000E000E000E000E000E000E000E000E000E000E007FE0>13 26 0 0 12] 102 dc[<0FCE187330307038703870387038303018602FC02000600070003FF03FFC1FFE600FC0 03C003C003C0036006381C07E0>16 24 1 8 19]103 dc[18 26 1 0 21]104 dc[<18003C00 3C001800000000000000000000000000FC001C001C001C001C001C001C001C001C001C001C001C 001C001C001C00FF80>9 26 0 0 10]105 dc[<018003C003C001800000000000000000000000 000FC001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C0 41C0E180E3007E00>10 33 -2 7 12]106 dc[18 26 1 0 20]107 dc[9 26 0 0 10]108 dc[29 16 1 0 32]109 dc[18 16 1 0 21]110 dc[<07E01C38300C 700E6006E007E007E007E007E007E0076006700E381C1C3807E0>16 16 1 0 19]111 dc[18 23 1 7 21]112 dc[< 03C2000C2600381E00300E00700E00E00E00E00E00E00E00E00E00E00E00E00E00700E00700E00 381E001C2E0007CE00000E00000E00000E00000E00000E00000E00007FC0>18 23 1 7 20]113 dc[12 16 1 0 15]114 dc[<1F2060E04020C020C020F0007F003FC01FE000F080708030C030C020F0408F80> 12 16 1 0 15]115 dc[<0400040004000C000C001C003C00FFC01C001C001C001C001C001C00 1C001C001C201C201C201C201C200E400380>11 23 1 0 15]116 dc[ 18 16 1 0 21]117 dc[17 16 1 0 20]118 dc[25 16 1 0 28]119 dc[18 16 1 0 20]120 dc[17 23 1 7 20]121 dc dfe /fj df[<001F83E000F06E3001C078780380F8780300F0300700 7000070070000700700007007000070070000700700007007000FFFFFF80070070000700700007 007000070070000700700007007000070070000700700007007000070070000700700007007000 0700700007007000070070000700700007007000070070007FE3FF00>29 32 0 0 27]11 dc[< 003F0000E0C001C0C00381E00701E00701E0070000070000070000070000070000070000FFFFE0 0700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E0 0700E00700E00700E00700E00700E07FC3FE>23 32 0 0 25]12 dc[<003FE000E0E001C1E003 81E00700E00700E00700E00700E00700E00700E00700E00700E0FFFFE00700E00700E00700E007 00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E007 00E00700E07FE7FE>23 32 0 0 25]13 dc[<001F81F80000F04F040001C07C06000380F80F00 0300F00F000700F00F000700700000070070000007007000000700700000070070000007007000 00FFFFFFFF00070070070007007007000700700700070070070007007007000700700700070070 070007007007000700700700070070070007007007000700700700070070070007007007000700 7007000700700700070070070007007007007FE3FE3FF0>36 32 0 0 38]14 dc[<001F81FF00 00F06F070001C07C0F000380F80F000300F0070007007007000700700700070070070007007007 00070070070007007007000700700700FFFFFFFF00070070070007007007000700700700070070 070007007007000700700700070070070007007007000700700700070070070007007007000700 7007000700700700070070070007007007000700700700070070070007007007007FE3FE3FF0> 36 32 0 0 38]15 dc[<07070F1C383060C0>8 8 9 -24 23]19 dc[<0E000F0003C000E000E0 00E001C07F808000>11 9 5 10 20]24 dc[<7038F87CFC7EFC7E743A04020402040208040804 1008100820104020>15 14 2 -18 23]34 dc[<0F0000301880003030600060703001C0602C06 C06013F980E0100300E0100300E0100600E0100C00E0100C00E010180060103000602030007020 60003040C0001880C0000F018000000300E0000303100006060800060604000C0C0400180C0400 181C0200301C0200601C0200601C0200C01C0201801C0201801C0203000C0406000C0406000604 0C00060818000310080000E0>31 37 3 3 38]37 dc[<70F8FCFC74040404080810102040>6 14 4 -18 13]39 dc[<0020004000800100020006000C000C00180018003000300030007000600060 006000E000E000E000E000E000E000E000E000E000E000E000E000600060006000700030003000 3000180018000C000C00060002000100008000400020>11 46 3 12 18]40 dc[<800040002000 100008000C00060006000300030001800180018001C000C000C000C000E000E000E000E000E000 E000E000E000E000E000E000E000C000C000C001C001800180018003000300060006000C000800 1000200040008000>11 46 3 12 18]41 dc[<0006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000000600 00FFFFFFF0FFFFFFF0000600000006000000060000000600000006000000060000000600000006 000000060000000600000006000000060000000600000006000000060000>28 32 3 5 35]43 dc[<70F8FCFC74040404080810102040>6 14 4 9 13]44 dc[10 2 1 -9 15]45 dc[<70F8F8F870>5 5 4 0 13]46 dc[<000100030003000600060006000C000C000C00180018 001800300030003000600060006000C000C000C00180018001800300030003000600060006000C 000C000C00180018001800300030003000600060006000C000C000C000>16 45 3 11 23]47 dc [<03F0000E1C001C0E00180600380700700380700380700380700380F003C0F003C0F003C0F003 C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07003807003807003807807 803807001806001C0E000E1C0003F000>18 31 2 1 23]48 dc[<018003800F80F38003800380 038003800380038003800380038003800380038003800380038003800380038003800380038003 800380038007C0FFFE>15 30 4 0 23]49 dc[<03F0000C1C00100E00200700400780800780F0 07C0F803C0F803C0F803C02007C00007C0000780000780000F00000E00001C0000380000700000 600000C0000180000300000600400C00401800401000803FFF807FFF80FFFF80>18 30 2 0 23] 50 dc[<03F0000C1C00100E00200F00780F80780780780780380F80000F80000F00000F00000E 00001C0000380003F000003C00000E00000F000007800007800007C02007C0F807C0F807C0F807 C0F00780400780400F00200E001C3C0003F000>18 31 2 1 23]51 dc[<000600000600000E00 000E00001E00002E00002E00004E00008E00008E00010E00020E00020E00040E00080E00080E00 100E00200E00200E00400E00C00E00FFFFF0000E00000E00000E00000E00000E00000E00000E00 00FFE0>20 30 1 0 23]52 dc[<1803001FFE001FFC001FF8001FE00010000010000010000010 000010000010000011F000161C00180E001007001007800003800003800003C00003C00003C070 03C0F003C0F003C0E00380400380400700200600100E000C380003E000>18 31 2 1 23]53 dc[ <007C000182000701000E03800C07801C0780380300380000780000700000700000F1F000F21C 00F40600F80700F80380F80380F003C0F003C0F003C0F003C0F003C07003C07003C07003803803 803807001807000C0E00061C0001F000>18 31 2 1 23]54 dc[<4000007FFFC07FFF807FFF80 40010080020080020080040000080000080000100000200000200000400000400000C00000C000 01C000018000038000038000038000038000078000078000078000078000078000078000078000 030000>18 31 3 1 23]55 dc[<03F0000C0C0010060030030020018060018060018060018070 01807803003E03003F06001FC8000FF00003F80007FC000C7E00103F00300F806003804001C0C0 01C0C000C0C000C0C000C0C000806001802001001002000C0C0003F000>18 31 2 1 23]56 dc[ <03F0000E18001C0C00380600380700700700700380F00380F00380F003C0F003C0F003C0F003 C0F003C07007C07007C03807C0180BC00E13C003E3C00003800003800003800007003007007806 00780E00700C002018001070000FC000>18 31 2 1 23]57 dc[<70F8F8F87000000000000000 00000070F8F8F870>5 20 4 0 13]58 dc[<70F8F8F8700000000000000000000070F0F8F87808 0808101010202040>5 29 4 9 13]59 dc[<7FFFFFE0FFFFFFF000000000000000000000000000 00000000000000000000000000000000000000FFFFFFF07FFFFFE0>28 12 3 -5 35]61 dc[<0F C0307040384038E03CF03CF03C603C0038007000E000C001800180010003000200020002000200 020002000000000000000000000007000F800F800F800700>14 32 3 0 21]63 dc[<00010000 0003800000038000000380000007C0000007C0000007C0000009E0000009E0000009E0000010F0 000010F0000010F00000207800002078000020780000403C0000403C0000403C0000801E000080 1E0000FFFE0001000F0001000F0001000F00020007800200078002000780040003C00E0003C01F 0007E0FFC03FFE>31 32 1 0 34]65 dc[26 31 2 0 32]66 dc[<000FC0 40007030C001C009C0038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00 004078000040F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8 000000780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003800200 01C0040000703800000FC000>26 33 3 1 33]67 dc[29 31 2 0 35]68 dc[27 31 2 0 31]69 dc[25 31 2 0 30]70 dc[<00 0FE0200078186000E004E0038002E0070001E00F0000E01E0000601E0000603C0000603C000020 7C00002078000020F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8007F FCF80003E0780001E07C0001E03C0001E03C0001E01E0001E01E0001E00F0001E0070001E00380 02E000E0046000781820000FE000>30 33 3 1 36]71 dc[29 31 2 0 34] 72 dc[14 31 1 0 16]73 dc[<0F FFC0007C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C0000 3C00003C00003C00003C00003C00003C00003C00003C00003C00003C00203C00F83C00F83C00F8 3C00F0380040780040700030E0000F8000>18 32 2 1 23]74 dc[30 31 2 0 35]75 dc[23 31 2 0 28]76 dc[37 31 2 0 42] 77 dc[29 31 2 0 34]78 dc[<001F800000F0F00001C0380007801E000F 000F000E0007001E0007803C0003C03C0003C07C0003E0780001E0780001E0F80001F0F80001F0 F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07C0003E07C0003 E03C0003C03C0003C01E0007800E0007000F000F0007801E0001C0380000F0F000001F8000>28 33 3 1 35]79 dc[25 31 2 0 31]80 dc[<001F800000F0F00001C03800 07801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C0003E0780001E0F80001 F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07800 01E07C0003E03C0003C03C0F03C01E1087800E2047000F204F0007A03E0001E0380000F0F01000 1FB01000003010000038300000387000003FF000001FE000001FE000000FC000000780>28 41 3 9 35]81 dc[30 32 2 1 33]82 dc[<07E0800C19801007803003 80600180600180E00180E00080E00080E00080F00000F000007800007F00003FF0001FFC000FFE 0003FF00001F800007800003C00003C00001C08001C08001C08001C08001C0C00180C00380E003 00F00600CE0C0081F800>18 33 3 1 25]83 dc[<7FFFFFE0780F01E0600F0060400F0020400F 0020C00F0030800F0010800F0010800F0010800F0010000F0000000F0000000F0000000F000000 0F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000001F800007FFFE00>28 31 2 0 33]84 dc[< FFFC3FF80FC007C007800380078001000780010007800100078001000780010007800100078001 000780010007800100078001000780010007800100078001000780010007800100078001000780 010007800100078001000780010007800100038002000380020001C0020001C0040000E0080000 70180000382000000FC000>29 32 2 1 34]85 dc[31 32 1 1 34]86 dc[44 32 1 1 47]87 dc[<7FF83FF80FE00FC007C0070003C0020001E0040001F00C0000F0 080000781000007C1000003C2000003E4000001E4000000F8000000F8000000780000003C00000 07E0000005E0000009F0000018F8000010780000207C0000603C0000401E0000801F0001800F00 01000780020007C0070003C01F8007E0FFE01FFE>31 31 1 0 34]88 dc[ 32 31 1 0 34]89 dc[<7FFFF87C00F87000F06001E04001E0C003C0C003C0800780800F80800F 00001E00001E00003C00003C0000780000F80000F00001E00001E00003C00403C0040780040F80 040F000C1E000C1E00083C00183C0018780038F801F8FFFFF8>22 31 3 0 28]90 dc[7 45 4 11 13]91 dc[<080410082010201040204020804080408040B85CFC7EFC7E7C 3E381C>15 14 5 -18 23]92 dc[7 45 1 11 13]93 dc[<1FE00030300078 1800781C00300E00000E00000E00000E0000FE00078E001E0E00380E00780E00F00E10F00E10F0 0E10F01E10781E103867200F83C0>20 20 2 0 23]97 dc[<0E0000FE00000E00000E00000E00 000E00000E00000E00000E00000E00000E00000E00000E3E000EC3800F01C00F00E00E00E00E00 700E00700E00780E00780E00780E00780E00780E00780E00700E00700E00E00F00E00D01C00CC3 00083E00>21 32 1 0 25]98 dc[<03F80E0C1C1E381E380C70007000F000F000F000F000F000 F00070007000380138011C020E0C03F0>16 20 2 0 20]99 dc[<000380003F80000380000380 00038000038000038000038000038000038000038000038003E380061B801C0780380380380380 700380700380F00380F00380F00380F00380F00380F003807003807003803803803807801C0780 0E1B8003E3F8>21 32 2 0 25]100 dc[<03F0000E1C001C0E00380700380700700700700380F0 0380F00380FFFF80F00000F00000F000007000007000003800801800800C010007060001F800> 17 20 1 0 20]101 dc[<007C00C6018F038F07060700070007000700070007000700FFF00700 070007000700070007000700070007000700070007000700070007000700070007007FF0>16 32 0 0 14]102 dc[<0000E003E3300E3C301C1C30380E00780F00780F00780F00780F00780F0038 0E001C1C001E380033E0002000002000003000003000003FFE001FFF800FFFC03001E0600070C0 0030C00030C00030C000306000603000C01C038003FC00>20 31 1 10 23]103 dc[<0E0000FE 00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E3E000E43000E 81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C0FFE7FC>22 32 1 0 25]104 dc[<1C001E003E001E001C0000000000 00000000000000000E007E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E00FFC0>10 31 0 0 12]105 dc[<00E001F001F001F000E0000000000000000000 000000007007F000F0007000700070007000700070007000700070007000700070007000700070 0070007000700070007000706070F060F0C061803F00>12 40 -2 9 14]106 dc[<0E0000FE00 000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0FF00E03C00E03 000E02000E04000E08000E10000E30000E70000EF8000F38000E1C000E1E000E0E000E07000E07 800E03800E03C00E03E0FFCFF8>21 32 1 0 24]107 dc[<0E00FE000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E00FFE0>11 32 0 0 12]108 dc[<0E1F01F000FE618618000E81C81C000F00F0 0E000F00F00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00 E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E00FF E7FE7FE0>35 20 1 0 38]109 dc[<0E3E00FE43000E81800F01C00F01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC>22 20 1 0 25]110 dc[<01F800070E001C03803801C03801C07000E07000E0F000F0F000F0F000F0F0 00F0F000F0F000F07000E07000E03801C03801C01C0380070E0001F800>20 20 1 0 23]111 dc [<0E3E00FEC3800F01C00F00E00E00E00E00F00E00700E00780E00780E00780E00780E00780E00 780E00700E00F00E00E00F01E00F01C00EC3000E3E000E00000E00000E00000E00000E00000E00 000E00000E0000FFE000>21 29 1 9 25]112 dc[<03E0800619801C05803C0780380380780380 700380F00380F00380F00380F00380F00380F003807003807803803803803807801C0B800E1380 03E380000380000380000380000380000380000380000380000380003FF8>21 29 2 9 24]113 dc[<0E78FE8C0F1E0F1E0F0C0E000E000E000E000E000E000E000E000E000E000E000E000E000E 00FFE0>15 20 1 0 18]114 dc[<1F9030704030C010C010C010E00078007F803FE00FF0007080 3880188018C018C018E030D0608F80>13 20 2 0 18]115 dc[<020002000200060006000E000E 003E00FFF80E000E000E000E000E000E000E000E000E000E000E000E080E080E080E080E080610 031001E0>13 28 1 0 18]116 dc[<0E01C0FE1FC00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E03C00603C0030DC001F1FC>22 20 1 0 25]117 dc[21 20 1 0 24]118 dc [30 20 1 0 33]119 dc[<7FC3FC0F01E00701C007018003810001C20000E40000EC0000 7800003800003C00007C00004E000087000107000303800201C00601E01E01E0FF07FE>23 20 0 0 24]120 dc[21 29 1 9 24]121 dc[<3FFF380E200E201C40384078 407000E001E001C00380078007010E011E011C0338027006700EFFFE>16 20 1 0 20]122 dc[< FFFFFC>22 1 0 -12 23]123 dc[44 1 0 -12 45]124 dc dfe /fk df[11 2 2 -9 17]45 dc[<018003801F80E3800380038003800380038003800380038003 8003800380038003800380038003800380038003800380038003800380038007C0FFFE>15 30 5 0 25]49 dc[<03F0000C1E001007002007804003C04003C0F003E0F801E0F801E0F801E02003E0 0003E00003C00007C0000780000F00000E00001C0000380000700000C000018000030000020020 0400200800201000403FFFC07FFFC0FFFFC0>19 30 3 0 25]50 dc[<03F8000C1E0010070020 07807807C07803C07803C03807C00007C0000780000780000F00000E0000380003F000001E0000 0F000007800007C00003C00003E02003E0F803E0F803E0F803E0F003C04007C0400780300F000C 1E0003F000>19 31 3 1 25]51 dc[<01F000060C001C06001803003803807003807001C0F001 C0F001C0F001E0F001E0F001E0F001E07001E07003E03803E03803E01C05E00E19E003E1E00001 C00001C00003C0000380300380780700780600700E00201C001030000FC000>19 31 3 1 25] 57 dc[<000FE01000381C3000E0027003C00170078000F00F0000701E0000701E0000303C0000 303C0000107C00001078000010F8000000F8000000F8000000F8000000F8000000F8000000F800 0000F8000000F8000000780000007C0000103C0000103C0000101E0000201E0000200F00002007 80004003C0008000E0030000380C00000FF000>28 33 4 1 36]67 dc[39 31 3 0 45]77 dc[ <03F0200C0C601803E03000E06000E0600060E00060E00020E00020E00020F00000F800007C00 007F80003FFC001FFF0007FF8001FFC0000FE00003E00001F00000F00000708000708000708000 70800070C00060C000E0E000C0F80180C6030081FC00>20 33 4 1 28]83 dc[31 32 3 1 37]85 dc dfe /fl df[<07070F1E1C38604080>8 9 10 -26 24]19 dc[<70F8F8F870>5 5 4 0 14]46 dc[<008003800F80F3800380038003800380038003800380 038003800380038003800380038003800380038003800380038003800380038003800380038003 8007C0FFFE>15 33 4 0 24]49 dc[<03F0000C1C001007002007804003C04003C08003E0F003 E0F801E0F801E0F801E02003E00003E00003C00003C0000780000700000E00001C000018000030 0000600000C0000180000100000200200400200800201800603000403FFFC07FFFC0FFFFC0>19 33 2 0 24]50 dc[<01F000060C000C0600180700380380700380700380F001C0F001C0F001C0 F001E0F001E0F001E0F001E0F001E07001E07003E03803E01805E00C05E00619E003E1E00001C0 0001C00001C0000380000380300300780700780600700C002018001030000FC000>19 34 2 1 24]57 dc[29 34 2 0 35]66 dc[<0007E010 0038183000E0063001C00170038000F0070000F00E0000701E0000701C0000303C0000303C0000 307C0000107800001078000010F8000000F8000000F8000000F8000000F8000000F8000000F800 0000F800000078000000780000107C0000103C0000103C0000101C0000201E0000200E00004007 0000400380008001C0010000E0020000381C000007E000>28 36 3 1 35]67 dc[<03FFF0001F 00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F 00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00700F00F80F 00F80F00F80E00F01E00401C0020380018700007C000>20 35 2 1 25]74 dc[40 34 2 0 45]77 dc[ 32 34 2 0 37]78 dc[<0FE0001838003C0C003C0E0018070000070000070000070000FF0007C7 001E07003C0700780700700700F00708F00708F00708F00F087817083C23900FC1E0>21 21 2 0 24]97 dc[<01FE000703000C07801C0780380300780000700000F00000F00000F00000F00000F0 0000F00000F000007000007800403800401C00800C010007060001F800>18 21 2 0 22]99 dc[ <0000E0000FE00001E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E001F8E00704E00C02E01C01E03800E07800E07000E0F000E0F000E0F000E0F000E0F000 E0F000E0F000E07000E07800E03800E01801E00C02E0070CF001F0FE>23 35 2 0 27]100 dc[< 01FC000707000C03801C01C03801C07801E07000E0F000E0FFFFE0F00000F00000F00000F00000 F000007000007800203800201C00400E008007030000FC00>19 21 1 0 22]101 dc[<0E0000FE 00001E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E 1F800E60C00E80E00F00700F00700E00700E00700E00700E00700E00700E00700E00700E00700E 00700E00700E00700E00700E00700E00700E0070FFE7FF>24 35 1 0 27]104 dc[<1C001E003E 001E001C00000000000000000000000000000000000E00FE001E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E00FFC0>10 34 1 0 14]105 dc[<0E00FE00 1E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E00FFE0>11 35 1 0 14]108 dc[<0E 1F80FE60C01E80E00F00700F00700E00700E00700E00700E00700E00700E00700E00700E00700E 00700E00700E00700E00700E00700E00700E0070FFE7FF>24 21 1 0 27]110 dc[<01FC000707 000C01801800C03800E0700070700070F00078F00078F00078F00078F00078F00078F000787000 707800F03800E01C01C00E038007070001FC00>21 21 1 0 24]111 dc[<0E3CFE461E8F0F0F0F 060F000E000E000E000E000E000E000E000E000E000E000E000E000E000F00FFF0>16 21 1 0 19]114 dc[<0F8830786018C018C008C008E008F0007F803FE00FF001F8003C801C800C800CC0 0CC008E018D0308FC0>14 21 2 0 19]115 dc[<02000200020002000600060006000E001E003E 00FFF80E000E000E000E000E000E000E000E000E000E000E000E040E040E040E040E040E040708 030801F0>14 31 1 0 19]116 dc[<0E0070FE07F01E00F00E00700E00700E00700E00700E0070 0E00700E00700E00700E00700E00700E00700E00700E00700E00F00E00F006017003827800FC7F >24 21 1 0 27]117 dc dfe /fm df[15 3 1 -12 20]45 dc[36 41 2 0 42]66 dc[40 41 2 0 45]72 dc[32 41 2 0 37]76 dc[40 41 2 0 45]78 dc[35 41 2 0 41]80 dc[<00FE010003FF83000F81E3001E0037003C001F 0038000F007800070070000700F0000300F0000300F0000300F0000100F8000100F8000100FC00 00007C0000007F0000003FE000001FFF00000FFFE00007FFF80003FFFC00007FFE000007FF0000 007F0000001F8000000F80000007C0000007C0800003C0800003C0800003C0800003C0C00003C0 C0000380C0000380E0000780F0000700F8000E00EE001C00C3C07800C1FFF000803FC000>26 43 3 1 33]83 dc[<01FC00000E0780001001C0003C00E0003E00F0003E0078001C00780008007800 000078000000780000007800007FF80003E078000F8078001F0078003E0078007C007800780078 20F8007820F8007820F8007820F800F8207C00F8203C013C401F063FC007F80F00>27 26 2 0 30]97 dc[<007F8001C0700780080F003C1E007C3C007C3C00387C0010780000F80000F80000F8 0000F80000F80000F80000F80000F800007800007C00003C00043C00041E00080F001007802001 C0C0007F00>22 26 2 0 27]99 dc[<00000F000001FF000001FF0000001F0000000F0000000F 0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F000000 0F00003F0F0001C0CF0003802F000F001F001E001F001C000F003C000F007C000F0078000F0078 000F00F8000F00F8000F00F8000F00F8000F00F8000F00F8000F00F8000F0078000F0078000F00 3C000F003C000F001E001F000E002F0007004F8001C18FF8007E0FF8>29 42 2 0 33]100 dc[< 007E0003C3800700E00E00F01C00703C00783C003878003C78003CF8003CF8003CFFFFFCF80000 F80000F80000F80000F800007800007C00003C00043C00041E00080E001007002001C0C0007F00 >22 26 2 0 27]101 dc[<001F000070C000E1E001C3E003C3E00381C007808007800007800007 8000078000078000078000078000078000078000FFFE00FFFE0007800007800007800007800007 800007800007800007800007800007800007800007800007800007800007800007800007800007 800007800007800007800007C000FFFE00FFFE00>19 42 1 0 18]102 dc[<0000078001FC1840 070721C00E03C1C01E03C0803C01E0003C01E0007C01F0007C01F0007C01F0007C01F0007C01F0 003C01E0003C01E0001E03C0000E0380001707000011FC00003000000030000000300000003800 00001C0000001FFFC0000FFFF80007FFFC001C003E0030000F007000070060000380E0000380E0 000380E0000380E0000380700007007000070038000E000C0018000780F00000FF8000>26 40 2 13 30]103 dc[<07800000FF800000FF8000000F80000007800000078000000780000007800000 07800000078000000780000007800000078000000780000007800000078000000783F800078C1C 0007900E0007A0070007A0078007C0078007C00780078007800780078007800780078007800780 078007800780078007800780078007800780078007800780078007800780078007800780078007 8007800780078007800780FFFCFFFCFFFCFFFC>30 42 1 0 33]104 dc[<07000F801F801F800F 80070000000000000000000000000000000000000007807F807F800F8007800780078007800780 078007800780078007800780078007800780078007800780078007800780FFF8FFF8>13 41 1 0 17]105 dc[<07800000FF800000FF8000000F8000000780000007800000078000000780000007 8000000780000007800000078000000780000007800000078000000780000007807FF007807FF0 07801F8007801C0007801800078020000780400007808000078100000782000007870000079F80 0007A7800007C7C0000783E0000781E0000781F0000780F8000780780007807C0007803E000780 1E0007801F0007801F80FFFC7FF8FFFC7FF8>29 42 1 0 32]107 dc[<0780FF80FF800F800780 078007800780078007800780078007800780078007800780078007800780078007800780078007 80078007800780078007800780078007800780078007800780078007800780FFFCFFFC>14 42 1 0 17]108 dc[<0781F800FC00FF860E030700FF98070C03800FA0079003C007A003D001E007C0 03E001E007C003E001E0078003C001E0078003C001E0078003C001E0078003C001E0078003C001 E0078003C001E0078003C001E0078003C001E0078003C001E0078003C001E0078003C001E00780 03C001E0078003C001E0078003C001E0078003C001E0078003C001E0078003C001E0FFFC7FFE3F FFFFFC7FFE3FFF>48 26 1 0 51]109 dc[<0783F800FF8C1C00FF900E000FA0070007A0078007 C0078007C007800780078007800780078007800780078007800780078007800780078007800780 078007800780078007800780078007800780078007800780078007800780078007800780FFFCFF FCFFFCFFFC>30 26 1 0 33]110 dc[<007F000001C1C000070070000E0038001C001C003C001E 003C001E0078000F0078000F00F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F800 0F80F8000F8078000F0078000F003C001E003C001E001E003C000E0038000700700001C1C00000 7F0000>25 26 2 0 30]111 dc[<0783F000FF8C1C00FFB00F0007A0078007C003C0078003E007 8001E0078001F0078001F0078000F8078000F8078000F8078000F8078000F8078000F8078000F8 078000F0078001F0078001F0078001E0078003C007C003C007C0078007A00E0007983C000787E0 000780000007800000078000000780000007800000078000000780000007800000078000000780 0000FFFC0000FFFC0000>29 38 1 12 33]112 dc[<0787C0FF98E0FF91F00FA1F007C1F007C0 E007C0000780000780000780000780000780000780000780000780000780000780000780000780 0007800007800007800007800007C000FFFE00FFFE00>20 26 1 0 23]114 dc[<07F8401C06C0 3001C06000C06000C0E00040E00040F00040F800007E00007FF0003FFE000FFF0003FF80003FC0 0007C08001E08001E0C000E0C000E0C000E0E000C0F001C0F80180C4070083F800>19 26 2 0 24]115 dc[<0080000080000080000080000180000180000180000380000380000780000F8000 1FFF80FFFF80078000078000078000078000078000078000078000078000078000078000078000 07800007800007804007804007804007804007804007804007804003C08001C08000E100003E00 >18 37 1 0 23]116 dc[40 26 1 0 43]119 dc[29 38 1 12 32] 121 dc dfe end TeXDict begin @letter %%EndProlog %%Page: 0 1 bop 573 529 a fm(Simple)18 b(Proto)q(col)h(Pro)r(cessing)f(for)421 604 y(Hig\ h-Bandwidth)i(Lo)n(w-Latency)g(Net)n(w)n(orking)522 737 y fl(Jos)o(\023)-22 b (e)16 b(C.)g(Brustoloni)138 b(Brian)16 b(N.)f(Bershad)850 808 y(Marc)o(h)h (1992)796 877 y fk(CMU-CS-93-132)698 1245 y fj(Sc)o(ho)q(ol)g(of)e(Computer)h (Science)703 1301 y(Carnegie)h(Mellon)g(Universit)o(y)760 1357 y(Pittsburgh,) e(P)m(A)h(15213)69 2527 y fi(This)20 b(researc)o(h)f(w)o(as)g(sp)q(onsored)h (b)o(y)f(The)g(Defense)g(Adv)n(anced)h(Research)f(Pro)r(jects)g(Agency)m(,)h (Information)g(Science)g(and)0 2572 y(T)m(ec)o(hnology)11 b(O\016ce,)e(under) i(the)e(title)i(\\Research)e(on)h(P)o(arallel)h(Computing",)h(ARP)m(A)d(Order) g(No.)g(7330,)h(issued)h(by)e(D)o(ARP)m(A/CMO)0 2618 y(under)14 b(Con)o(tract) f(MD)o(A972-90-C-0035.)69 2694 y(The)f(views)i(and)f(conclusions)i(contained) e(in)h(this)f(do)q(cument)f(are)h(those)g(of)f(the)h(authors)g(and)g(should)h (not)f(b)q(e)g(interpreted)g(as)0 2740 y(representing)h(the)f(o\016cial)i(p)q (olicies,)g(either)f(expressed)g(or)f(implied,)i(of)e(D)o(ARP)m(A)g(or)g(the) g(U.S.)f(gov)o(ernment.)p eop %%Page: 1 2 bop 62 2672 a fh(Keyw)n(ords:)18 b fj(A)m(TM,)12 b(high-sp)s(eed)i(net)o(w)n (orks,)f(ILP)m(,)h(protocol)g(b)o(ypass,)f(proto)q(col)h(pro)q(cessing,)g(bu\ \013er)g(man-)0 2728 y(agemen)o(t,)g(\015o)o(w)h(con)o(trol,)f(Mac)o(h,)h(TCP\ /IP)p eop %%Page: 2 3 bop 875 873 a fh(Abstract)0 979 y fj(Protocol)23 b(and)g(op)q(erating)g(syst\ em)f(o)n(v)o(erheads)h(ha)n(v)o(e)g(b)q(ecome)g(the)f(limiting)j(factor)c(for) h(comm)o(unications)0 1036 y(p)r(erformance)14 b(on)g(fast)g(net)o(w)n(orks)g (suc)o(h)h(as)f(A)m(TM.)f(A)i(large)f(comp)r(onen)o(t)g(of)g(these)h(o)n(v)o (erheads)g(stems)f(from)g(the)0 1092 y fg(pr)n(oto)n(c)n(ol)e(r)n(e)m(dundanc\ y)f fj(that)f(arises)h(when)h(la)o(y)n(ering)g(higher)f(level)g(proto)q(cols) g(on)g(top)f(of)h(lo)o(w)n(er)g(level)g(ones.)19 b(A)l(TM,)0 1149 y(for)i(exa\ mple,)i(requires)f(sp)q(eci\014c)h(mec)o(hanisms)f(for)f(connection)h(managem\ en)o(t,)g(\015o)n(w)g(con)n(trol,)h(congestion)0 1205 y(a)n(v)o(oidance,)12 b (and)f(segmen)o(tation)g(and)g(reassem)n(bly)m(.)19 b(With)11 b(these)g(mec)o (hanisms)g(in)h(place,)g(it)f(is)h(relativ)o(ely)f(simple)0 1262 y(and)18 b (inexp)s(ensiv)o(e)g(to)g(pro)n(vide)h(for)e(reliable)j(sequenced)g(delivery) e(at)f(the)h(net)o(w)o(ork)f(interface)g(level,)i(making)0 1318 y(similar)12 b (functionalit)q(y)f(in)g(higher-lev)q(el)h(protocols)g(suc)o(h)f(as)g(TCP/IP) g(redundan)o(t.)19 b(In)11 b(this)h(pap)q(er)f(w)o(e)g(present)f(a)0 1375 y (protocol)k(arc)n(hitecture)g(sp)r(eci\014cally)h(tailored)e(for)g(comm)n(uni\ cation)h(o)o(v)n(er)f(high-bandwidth)i(lo)o(w-latency)f(lo)q(cal)0 1431 y(or) g(metrop)q(olitan)i(A)l(TM)e(net)o(w)o(orks.)19 b(Our)c(arc)n(hitecture)h(yie\ lds)h(high)e(p)r(erformance)f(b)o(y)h(eliminating)i(protocol)0 1487 y(redunda\ ncy)23 b(and)e(b)o(y)h(exploiting)h(common-case)e(comm)n(unication)i(b)r(eha) o(vior.)38 b(With)22 b(this)g(approac)n(h,)h(w)o(e)0 1544 y(can)d(com)n(bine) i(the)d(functionalit)q(y)g(t)o(ypically)j(found)e(in)g(four)g(separate)f(la)o (y)o(ers)h(of)f(the)h(ISO)h(mo)q(del)f(|)h(data)0 1600 y(link)c(through)d(ses\ sion)i(|)g(in)g(a)f(single)h(pass)f(o)o(v)n(er)g(the)h(data,)e(delivering)i (high)g(throughput)f(and)g(low)f(latency)m(.)0 1657 y(Our)e(protocol)g(arc)o (hitecture)g(requires)g(minimal)h(hardw)o(are)e(supp)r(ort)g(from)f(the)i(net) o(w)n(ork)f(interface)g(and)h(switc)o(h)0 1713 y(fabric,)g(y)o(et)f(e\016cien\ tly)g(pro)o(vides)h(services)g(suc)o(h)f(as)g(segmen)o(tation)g(and)g(reassem) n(bly)m(,)h(\015o)o(w)e(con)o(trol,)i(congestion)0 1770 y(a)n(v)o(oidance,)18 b(and)f(error)e(reco)o(v)o(ery)l(.)24 b(W)l(e)17 b(ha)o(v)n(e)g(implemen)q (ted)f(our)h(protocol)g(arc)o(hitecture)g(on)g(a)f(switc)o(h-based)0 1826 y (A)l(TM)j(net)o(w)o(ork)g(consisting)i(of)e(DECstation)g(5000/200)e(w)o(orkst\ ations)h(running)j(the)f(Mac)n(h)g(3.0)f(op)q(erating)0 1883 y(system.)29 b (Our)18 b(implemen)q(tation)g(ac)o(hiev)o(es)h(latencies)g(and)g(bandwidths)g (close)g(to)f(the)g(ph)o(ysical)i(limitations)0 1939 y(imp)r(osed)15 b(b)o (y)g(the)h(hardw)n(are,)f(y)o(et)g(o\013ers)f(applications)j(a)e(high-level)h (reliable)h(transp)q(ort)e(in)o(terface.)p eop %%Page: 1 4 bop 0 42 a fh(1.)52 b(In)n(tro)r(duction)0 184 y fj(Net)o(w)n(orks)14 b(suc) o(h)h(as)g(A)l(TM)f([22)o(])h(and)f(HiPPI)i([13)o(])e(are)h(b)q(ecoming)h(inc\ reasingly)g(imp)r(ortan)n(t)f(due)g(to)f(their)h(high)0 241 y(bandwidths)h (and)g(lo)o(w)f(latency)m(.)k(High)d(bandwidth)g(is)g(imp)r(ortan)n(t)f(for)g (applications)h(suc)o(h)g(as)f(those)g(found)g(in)0 297 y(sup)r(ercomputing,) k(visualization)h(and)f(m)n(ultimedia.)32 b(Lo)n(w)18 b(latency)h(is)g(k)o (ey)g(to)e(e\016cient)h(remote)g(pro)q(cedure)0 354 y(call)f(supp)r(ort)d(and) i(\014ne-grain)h(distributed)g(computation.)j(Bandwidth)d(and)f(latency)g(gai\ ns)g(in)g(the)g(ph)o(ysical)0 410 y(medium,)g(ho)o(w)n(ev)o(er,)f(do)g(not)g (necessarily)h(result)g(in)g(corresp)q(ondingly)h(b)q(etter)e(comm)n(unicatio\ n)i(p)q(erformance)0 467 y(for)d(end)h(applications)h([27)o(].)j(The)14 b(w)o (ork)g(describ)r(ed)h(in)g(this)g(pap)q(er)f(is)h(targeted)f(at)f(pro)o(vidin\ g)j(to)d(applications)0 523 y(net)o(w)n(ork)k(latencies)i(and)e(bandwidths)h (close)g(to)f(inherent)g(hardw)o(are)g(limits)h(for)f(an)g(A)l(TM)g(net)o(w)o (ork.)25 b(T)l(o)17 b(do)0 580 y(this,)h(w)o(e)g(b)n(ypass)g(higher-level)h (protocols)f(b)r(ecause)g(the)f(functionalit)q(y)g(found)h(in)g(those)g(proto\ cols,)g(such)f(as)0 636 y(TCP/IP)12 b([24)o(],)g(is)h(made)f(redundant)f(b)o (y)i(services)g(required)g(in)g(the)f(lo)o(w)o(er-level)h(net)o(w)n(ork)f(in) o(terface.)19 b(A)l(TM,)12 b(for)0 693 y(example,)j(requires)h(sp)r(eci\014c) g(supp)r(ort)e(for)g(connection)i(managemen)o(t,)e(\015o)o(w)h(con)o(trol,)f (congestion)h(a)o(v)n(oidance,)0 749 y(and)g(segmen)o(tation)g(and)h(reassem) n(bly)m(,)f(making)g(similar)i(TCP/IP)d(functionalit)q(y)h(redundan)o(t.)62 847 y(Our)f(protocols)g(and)g(in)o(terfaces)f(are)g(designed)i(from)d(scratc) o(h)h(to)g(a)n(v)o(oid)h(biases)g(detrimental)f(to)g(A)l(TM)g(that)0 903 y (fa)n(v)o(or)19 b(obsolete)h(or)f(extraneous)g(arc)o(hitectures)h(or)f(proto) q(cols.)33 b(This)20 b(approac)o(h)g(is)g(not)f(new:)29 b(most)19 b(of)g(the) 0 960 y(w)o(ork)13 b(on)g(fast)g(RPC,)g(for)g(example,)i(is)f(la)o(y)n(ered)g (on)g(top)f(of)g(assumptions)h(ab)q(out)f(the)h(arc)o(hitecture)g(or)f(net)o (w)n(ork)0 1016 y(used)k([15)o(,)e(32].)21 b(Our)c(design)g(requires)f(only)h (minimal)g(arc)o(hitectural)g(supp)r(ort)e(from)g(the)h(host)g(in)o(terface)g (and)0 1073 y(switc)o(hes,)g(fa)n(v)o(oring)g(solutions)g(that)f(w)o(ork)g (on)g(hardw)o(are)g(inexp)s(ensive)h(enough)g(to)f(come)g(pre-con\014gured)i (on)0 1129 y(ev)o(ery)e(w)o(orkstation,)f(m)n(uc)o(h)i(as)f(Ethernet)g([21)o (])g(curren)o(tly)g(do)r(es.)k(Sp)r(eci\014cally)o(,)c(in)h(our)f(implementat\ ion:)68 1277 y ff(\017)23 b fg(we)f(optimize)h(our)g(pr)n(oto)o(c)m(ols)f(for) h(the)g(c)m(ommon)g(c)n(ase)e fj([18])g(of)h(lo)o(w-latency)m(,)h(high-bandwi\ dth)h(com-)114 1334 y(m)n(unication)c(o)o(v)o(er)f(an)f(A)m(TM)g(net)o(w)o (ork)g(scaling)i(up)f(to)g(metropolitan)h(distances)g(|)f(roughly)g(40)g(km.) 114 1390 y(Assuming)14 b(that)f(the)h(comm)n(unicating)h(hosts)e(are)g(connec\ ted)i(directly)g(through)e(A)l(TM)g(rather)h(than)f(an)114 1446 y(in)o(ternet) o(w)n(ork)k(reduces)g(the)g(complexity)g(of)g(the)g(net)n(w)o(ork)f(proto)q (col)h(soft)n(w)o(are.)24 b(Metrop)q(olitan)17 b(ranges)114 1503 y(imp)r(ose) c(only)i(mo)q(derate)e(propagation)h(dela)o(ys,)g(making)g(it)g(p)r(ossible)h (to)e(use)h(simple)h(proto)q(cols)f(for)f(\015o)o(w)114 1559 y(con)o(trol,)21 b(congestion)g(a)o(v)n(oidance)h(and)f(error)f(reco)o(v)n(ery)m(.)36 b(In)o (ternet)o(w)o(orking)20 b(and)h(routing)g(are)g(left)g(to)114 1616 y(higher-l\ evel)16 b(soft)n(w)o(are)f(that)f(is)i(in)o(v)o(ok)o(ed)f(on)g(an)g(exception) i(basis.)68 1710 y ff(\017)23 b fg(we)17 b(exploit)h(inte)m(gr)n(ate)n(d)g (layer)f(pr)n(o)n(c)n(essing)e fj([7)o(])h(b)r(ecause)h(eac)o(h)g(separate)f (protocol)i(la)o(y)n(er)f(can)g(tak)n(e)g(time)114 1766 y(comparable)g(to)f (that)g(required)i(to)f(transport)g(the)f(data)h(o)n(v)o(er)g(a)f(high-sp)s (eed)h(net)o(w)o(ork.)24 b(F)l(or)16 b(the)h(host)114 1823 y(to)e(k)n(eep)i (up)f(with)g(the)f(net)o(w)o(ork,)g(la)o(y)n(ers)h(m)n(ust)g(b)q(e)g(com)o (bined)h(or)e(eliminated.)23 b(Our)16 b(net)o(w)o(ork)f(in)o(terface)114 1879 y(is)h(organized)g(in)h(a)e(single)i(la)o(y)o(er,)f(and)g(requires)g(that)f (eac)o(h)h(data)f(b)o(yte)h(b)q(e)g(accessed)h(only)f(once)g(on)g(the)114 1935 y(path)f(b)q(et)o(w)n(een)h(application)h(and)e(the)h(ph)o(ysical)g(medium.) 68 2029 y ff(\017)23 b fg(we)d(inte)m(gr)n(ate)h(bu\013er)f(management)g fj (for)f(applications)i(and)f(proto)q(col)g(pro)q(cessing.)34 b(Using)20 b(the) g(same)114 2086 y(bu\013ers)c(for)h(b)q(oth,)f(eac)o(h)h(data)g(b)o(yte)f(nee\ ds)i(to)e(b)q(e)h(mo)o(v)n(ed)h(only)f(once)g(b)r(et)n(w)o(een)g(the)g(net)o (w)o(ork)f(and)h(the)114 2142 y(application's)f(address)f(space,)g(minimizing) j(data)d(cop)o(ying.)62 2290 y(Although)h(our)f(discussion)i(focuses)f(on)f (A)l(TM)g(net)o(w)n(orks,)f(man)o(y)h(of)g(the)g(tec)o(hniques)h(w)o(e)f(pres\ en)o(t)g(are)g(also)0 2346 y(applicable)j(to)c(other)h(high-sp)r(eed)h(net)o (w)n(orks,)f(including)i(those)e(found)h(in)g(tigh)o(tly)f(coupled)i(m)n(ulti\ computers.)0 2403 y(F)l(or)h(example,)h(our)f(bu\013er)g(managemen)n(t)g(and) h(segmen)n(tation)f(and)h(reassem)n(bly)g(approac)n(hes)g(could)g(b)r(e)f(ap-) 0 2459 y(plied)f(to)e(the)h(fat-trees)f(used)h(in)g(the)g(CM-5,)e(and)i(our)f (\015o)o(w)h(con)n(trol)g(and)g(congestion)f(a)o(v)o(oidance)h(algorithms)0 2516 y(could)g(b)r(e)f(used)h(in)g(the)f(P)o(aragon)f(mesh,)h(where)g(receive\ r)g(o)n(v)o(errun)g(is)h(b)r(ecoming)f(an)g(increasingly)i(imp)r(ortan)n(t)0 2572 y(problem)f([5)o(].)964 2844 y(1)p eop %%Page: 2 5 bop 0 42 a fh(The)18 b(rest)f(of)g(this)h(pap)r(er)0 169 y fj(The)e(rest)g (of)g(this)g(pap)r(er)g(is)g(organized)h(as)f(follo)o(ws.)22 b(In)17 b(Sectio\ n)g(2)f(w)n(e)h(pro)n(vide)g(some)f(bac)o(kground)g(on)g(A)m(TM)0 225 y(net)o (w)n(orking)f(and)f(proto)q(cols,)g(and)h(describ)r(e)g(the)f(hardw)o(are)g (w)n(e)h(used)g(in)g(our)f(w)o(ork.)19 b(In)c(Section)g(3)f(w)n(e)h(discuss)0 282 y(the)g(di\016culties)j(associated)e(with)g(the)f(traditional)h(la)o(y)o (ering)g(of)e(higher-lev)q(el)i(protocols)g(on)f(top)g(of)g(A)l(TM.)f(In)0 338 y(Section)19 b(4,)g(w)o(e)f(describ)s(e)h(the)f(o)o(v)n(erall)i(organization) f(and)f(implemen)q(tation)g(of)g(our)h(protocol)g(arc)n(hitecture,)0 395 y (in)f(particular)f(our)g(approac)n(h)g(to)f(bu\013er)h(managemen)o(t,)f(segme\ n)o(tation)h(and)g(reassem)n(bly)m(,)g(\015o)o(w)f(con)o(trol)h(and)0 451 y (error)h(reco)n(v)o(ery)l(,)h(signaling,)h(m)n(ultiplexing,)i(and)c(the)g(int\ egration)g(with)g(Internet)g(protocols.)29 b(In)19 b(Section)g(5)0 508 y(w)o (e)c(discuss)i(p)q(erformance.)j(In)c(Section)h(6)e(w)n(e)h(surv)n(ey)g(relat\ ed)g(w)n(ork.)k(Finally)n(,)15 b(in)h(Section)h(7)e(w)n(e)h(presen)o(t)f(our) 0 564 y(conclusions.)0 730 y fh(2.)52 b(Bac)o(kground)0 873 y fj(A)l(TM)13 b (is)i(the)f(in)o(ternational)g(standard)f(c)o(hosen)h(for)g(BISDN,)g(the)f (prop)r(osed)h(global)g(\014b)r(er-optic)g(net)o(w)n(ork)g(that)0 930 y(integ\ rates)d(and)i(replaces)g(the)g(curren)o(t)f(separate)g(net)o(w)o(orks)g(for)f (data,)h(v)o(oice)h(and)g(video)g(comm)o(unication)h([22)o(].)0 986 y(A)l(TM) j(pac)o(k)o(ets)f(\()p fg(c)n(el)q(ls)p fj(\))g(ha)o(v)o(e)h(a)g(\014xed)h (length)f(of)g(53)g(b)o(ytes.)25 b(Of)17 b(these,)h(5)f(b)o(ytes)f(are)h(the) g(A)m(TM)f fg(he)n(ader)i fj(with)0 1043 y(the)c(virtual)g(path)g(identi\014e\ r)h(and)f(virtual)g(connection)h(identi\014er)g(\(VPI/V)n(CI\))f(used)h(to)e (route)g(the)h(cell)i(to)d(the)0 1099 y(destination.)21 b(The)15 b(other)g (48)g(b)o(ytes)g(of)f(the)i(cell)g(are)f(the)h fg(p)n(aylo)m(ad)p fj(,)g(whic) o(h)g(carries)f(user)g(data.)62 1197 y(Messages)k(are)g(brok)o(en)h(up)g(in)o (to)f(cells)i(according)g(to)d(one)i(of)f(sev)o(eral)h(A)l(TM)f(adaptation)g (lay)n(er)h(options)0 1254 y(\(AAL\).)d(Our)g(hardw)o(are)g(supp)r(orts)f(AAL) i(3/4.)26 b(In)17 b(this)h(option,)g(the)f(A)m(TM)f(pa)o(yload)i(\()p fg(se)l (gment)p fj(\))f(is)h(divided)0 1310 y(into)h(a)g(2)g(b)o(yte)h(segmen)o(tati\ on)f(and)h(reassem)n(bly)h(\(SAR\))e(header,)i(44)e(b)o(ytes)g(of)h(data,)f (and)h(a)f(2)h(b)o(yte)f(SAR)0 1367 y(trailer.)39 b(The)22 b(SAR)h(header)f (con)n(tains)g(t)o(w)n(o)f(bits)h(to)f(indicate)i(segmen)o(t)f(t)n(yp)r(e)f (\(b)q(eginning,)k(con)o(tin)o(uation,)0 1423 y(end)17 b(of)f(message,)f(or)h (single)h(segmen)o(t)f(message\),)g(a)f(mo)q(dulo-16)i(segmen)o(t)f(sequence) h(n)o(um)o(b)r(er,)f(and)g(a)g(10)f(bit)0 1480 y(message)g(identi\014er)i(\(M\ ID\).)d(The)i(SAR)h(trailer)f(con)o(tains)g(the)g(segmen)o(t)g(c)n(hecksum)g (and)g(length)g(of)g(the)g(data)0 1536 y(in)h(the)f(segmen)o(t.)23 b(A)16 b (con)o(v)o(ergence)h(subla)o(y)o(er)f(\(CS\))g(runs)g(on)h(top)f(of)f(SAR)i (and)g(inserts)f(and)h(remo)n(v)o(es)f(a)g(four)0 1592 y(b)o(yte)i(header)h (and)f(a)h(four)e(b)o(yte)i(trailer)f(in)i(messages.)28 b(The)19 b(CS)f(heade\ r)h(con)o(tains)g(the)f(message)g(t)n(yp)r(e,)h(the)0 1649 y(lo)o(w)o(er)14 b (8)h(bits)g(of)f(the)h(MID,)f(and)g(the)h(size)h(of)e(the)h(bu\013er)f(requir\ ed)i(for)e(the)h(message.)k(The)c(CS)g(trailer)g(carries)0 1705 y(a)g(b)o(yte) g(for)f(proto)q(col)i(use,)f(the)g(lo)o(w)o(er)g(8)g(bits)g(of)g(the)g(MID,)g (and)g(the)h(total)e(message)h(length.)62 1803 y(T)l(o)20 b(transmit)f(a)g (message,)h(the)f(CS)h(header)g(and)g(trailer)g(are)f(inserted,)i(and)f(the)g (resulting)g(message)f(is)0 1860 y(brok)o(en)14 b(up)g(in)o(to)f(segmen)o(ts) g(eac)o(h)h(44)f(b)o(ytes)g(long,)h(except)g(p)r(ossibly)h(for)e(the)g(last)h (one,)f(which)h(ma)n(y)g(need)g(to)f(b)r(e)0 1916 y(padded)i(with)g(zeros)g (to)f(complete)h(a)f(segmen)o(t.)20 b(Segmen)o(ts)14 b(are)g(sen)o(t)h(in)g (individual)j(cells)e(to)e(the)g(destination,)0 1973 y(where)h(they)h(are)f (reassem)n(bled)h(into)f(the)g(original)h(message.)62 2071 y(Ev)o(ery)d(host) g(in)g(an)g(A)m(TM)f(net)o(w)o(ork)g(is)i(connected)g(to)e(one)i(or)e(more)h (switc)o(hes)g(that)f(route)h(pac)o(k)o(ets)g(b)q(et)o(w)n(een)0 2127 y(hosts) j(and)i(other)e(switc)o(hes.)26 b(In)17 b(con)o(trast)f(to)g(net)o(w)o(orks)g (suc)o(h)i(as)e(Ethernet,)h(a)g(connection)h(m)o(ust)e(b)r(e)h(estab-)0 2184 y (lished)i(b)r(et)n(w)o(een)f(an)o(y)f(pair)h(of)f(hosts)g(through)g(in)o(term\ ediate)h(switc)o(hes)f(b)r(efore)g(they)h(can)f(comm)o(unicate.)27 b(A)0 2240 y(connection)15 b(is)g(established)g(or)f(torn)f(do)o(wn)h(using)h(sp)r(eci\ \014c)g(signaling)h(protocols)f(running)g(on)f(top)f(of)h(CS,)g(and)0 2296 y (is)j(identi\014ed)h(b)o(y)e(the)h(VPI/V)o(CI)g(\014elds)h(of)e(the)g(A)m(TM) g(header.)24 b(Cells)18 b(from)e(a)g(giv)o(en)h(connection)h(are)e(routed)0 2353 y(o)n(v)o(er)f(a)f(path)g(determined)i(at)e(connection)h(establishment)f (time,)h(and)g(cannot)f(b)q(e)h(misordered)g(or)f(duplicated)0 2409 y(b)o(y)k (the)f(net)o(w)o(ork.)27 b(A)18 b(connectionless)h(service)g(has)e(also)h(b)r (een)g(prop)q(osed)g(for)f(A)m(TM,)f(but)i(its)g(sp)r(eci\014cations)0 2466 y (and)13 b(exact)f(c)o(haracteristics)h(are)g(still)h(in)g(progress.)k(As)13 b (with)g(other)f(\014b)r(er-optic)h(net)o(w)o(orks,)f(bit)h(error)f(rates)g (are)0 2522 y(extremely)17 b(lo)o(w)f(\(on)g(the)g(order)g(of)g(10)674 2506 y fe(\000)p fd(14)738 2522 y fj(\).)22 b(Cells)c(ma)n(y)l(,)e(ho)o(w)n(ev)o(er,) g(b)r(e)g(lost)g(due)h(to)f(corruption,)g(congestion,)0 2579 y(and)e(bu\013er) h(depletion)h(at)d(intermediate)h(switc)o(hes)h(and)f(endp)r(oint)f(no)r(des.) 20 b(Neither)15 b(the)f(A)l(TM)g(hardw)n(are)g(nor)0 2635 y(the)h(adaptation) g(la)o(y)o(ers)g(pro)o(vide)h(for)e(automatic)h(ac)o(kno)n(wledgements)g(and) g(retransmission.)964 2844 y(2)p eop %%Page: 3 6 bop 0 42 a fh(2.1.)52 b(Exp)q(erimen)n(tal)19 b(hardw)n(are)0 169 y fj(Our) 13 b(proto)q(col)g(arc)o(hitecture)g(is)h(designed)g(to)e(function)i(w)o(ell) g(using)g(simple)g(net)o(w)n(ork)f(in)o(terfaces)g(and)g(switc)o(hes.)0 225 y (Suc)o(h)i(con\014gurations)f(are)g(likely)h(to)e(b)r(ecome)h(ubiquitous)h (b)r(ecause)g(of)e(the)h(lo)o(w-cost)g(promised)h(b)o(y)f(their)g(sim-)0 282 y (plicit)q(y)l(.)26 b(The)17 b(system)g(describ)r(ed)h(in)g(this)f(pap)r(er)g (uses)h(a)e(F)l(ore)h(Systems')g(ASX-111)g(A)l(TM)f(switc)o(h)i([10)o(],)f (and)0 338 y(140)e(Mbps)g(TCA-100)g(TURBOchannel)i(A)l(TM)e(computer)g(interf\ aces)g([11)o(].)21 b(The)16 b(in)o(terfaces)1653 322 y fd(1)1688 338 y fj(lin\ k)h(DECsta-)0 395 y(tion)f(5000/200)e(w)o(orkstations)h([8)o(])h(that)f(use)i (a)e(25)h(MHz)g(MIPS)g(R3000)g(CPU)g([16)o(])f(running)j(o)n(v)o(er)e(a)g(25) f(MHz)0 451 y(TURBOchannel)h(I/O)g(bus)g([9)o(].)62 549 y(The)22 b(F)l(ore)f (Systems')f(interface)h(do)q(es)g(not)g(ha)o(v)o(e)g(its)h(o)n(wn)f(CPU,)g (and)h(do)q(es)f(not)g(supp)r(ort)f(DMA.)h(The)0 606 y(interface)10 b(p)r(erf\ orms)g(only)i(cell)g(delineation,)i(FIF)o(O)d(bu\013ering,)h(c)o(hec)o(ksummi\ ng,)h(and)e(in)o(terrupt)g(managemen)o(t.)0 662 y(W)l(e)16 b(b)q(eliev)q(e)f (that)g(this)i(is)f(the)f(minimum)h(supp)r(ort)f(required)i(from)e(an)n(y)h (A)l(TM)f(host)h(in)o(terface.)21 b(Ev)o(en)16 b(at)f(140)0 718 y(Mbps,)f(it) h(is)f(not)g(feasible)i(to)e(in)o(terrupt)g(the)h(host)e(CPU)i(with)f(eac)o (h)g(cell's)i(arriv)m(al)f(as)f(the)g(time)h(b)q(et)o(w)o(een)f(cells)0 775 y (is)i(only)f(ab)r(out)g(3)f(microseconds.)62 873 y(The)h(host)f(CPU)g(is)h (presen)o(ted)g(with)f(a)h(FIF)n(O)g(in)o(terface)f(through)g(which)h(it)f (sends)h(or)f(receives)g(A)m(TM)f(cells)0 929 y(using)i(programmed)f(I/O.)g (This)h(requires)g(that)f(eac)o(h)h(w)n(ord)f(in)i(the)e(cell)i(b)r(e)e(indiv\ iduall)q(y)j(written)e(or)f(read)g(b)o(y)0 986 y(the)h(host)f(CPU.)g(There)h (is)g(not)f(a)g(clear)h(case)g(for)f(or)g(against)g(DMA-supp)r(ort)f(in)j(lo) o(w-latency)f(net)o(w)o(orks.)k(The)0 1042 y(use)g(of)f(programmed)g(I/O)h (can)g(reduce)g(latency)h(since)f(no)g(DMA)f(set-up)h(is)g(required,)i(and)d (can)h(eliminate)0 1099 y(cac)o(he)13 b(consistency)g(problems)g(since)g(all) h(messages)d(go)h(through)g(the)g(CPU.)g(Ho)o(w)n(ev)o(er,)h(programmed)e(I/O) i(also)0 1155 y(limits)19 b(the)f(a)n(v)n(ailable)h(bandwidth)g(across)d(the) i(I/O)g(bus.)28 b(With)18 b(DMA)f(the)g(TURBOchannel)i(can)f(transfer)0 1212 y (one)e(32-bit)g(w)n(ord)g(p)q(er)g(cycle,)g(giving)h(a)e(p)r(eak)g(rate)g(of) h(800)e(Mbps.)22 b(In)16 b(con)o(trast,)e(programmed)h(I/O)h(requires)0 1268 y (8)g(cycles)i(to)d(read)i(a)f(w)n(ord)466 1252 y fd(2)486 1268 y fj(.)23 b (This)17 b(yields)h(a)e(ra)n(w)h(input)g(bandwidth)g(of)f(80)g(Mbps)h(for)e (the)i(CPU)f(to)g(transfer)0 1324 y(data)e(from)g(the)h(receive)g(FIF)n(O)g (to)g(memory)-5 b(,)15 b(whic)o(h)g(is)h(only)f(ab)q(out)g(half)g(of)f(the)h (net)o(w)o(ork)f(transmission)h(rate.)0 1381 y(On)g(the)g(transmit)f(path,)g (the)h(CPU)g(needs)g(3)f(cycles)i(to)e(write)h(w)o(ords)f(across)g(the)h(I/O) g(bus,)f(giving)i(a)e(p)r(eak)g(of)0 1437 y(160)f(Mbps)g(for)h(the)f(host)g (to)g(read)h(from)f(memory)g(and)h(write)g(to)f(the)g(transmit)g(FIF)o(O)h (|)g(enough)g(to)f(saturate)0 1494 y(the)18 b(link.)29 b(As)18 b(a)g(result,) h(the)f(receiv)o(e)g(path)g(will)i(tend)e(to)f(o)o(v)n(errun)h(when)h(driv)o (en)g(b)o(y)e(a)h(similarly)i(equipp)s(ed)0 1550 y(host,)c(so)h(\015o)n(w)g (con)o(trol)f(b)r(et)n(w)o(een)h(the)g(sender)h(and)e(receiver)h(is)g(needed) h(to)e(a)o(v)o(oid)h(loss)g(of)f(cells.)26 b(The)17 b(receive)0 1607 y(FIF)n (O)f(has)f(capacit)o(y)g(for)g(292)f(cells,)j(while)f(the)g(transmit)e(FIF)o (O)h(can)h(hold)g(up)f(to)g(36)f(cells.)62 1705 y(Eac)o(h)19 b(interface)g (is)h(connected)g(to)f(a)g(switc)o(h,)h(which)f(has)g(capacity)g(for)g(up)g (to)g(16)g(input)h(p)r(orts)e(and)i(16)0 1761 y(output)g(p)r(orts)f(\(switc)o (hes)h(with)h(64)f(p)q(orts)g(are)g(rapidly)i(b)q(ecoming)f(a)n(v)n(ailable\)\ .)37 b(Eac)n(h)21 b(output)f(p)r(ort)f(has)h(a)0 1818 y(FIF)n(O)d(with)g(capa\ cit)o(y)g(for)e(256)h(cells,)i(and)e(shares)h(with)f(three)h(other)f(output)g (p)r(orts)f(t)o(w)n(o)h(additional)i(FIF)o(Os)0 1874 y(with)e(eac)o(h)g(a)f (capacit)o(y)g(of)g(512)g(cells.)23 b(A)15 b(giv)o(en)h(output)g(p)q(ort)f (has,)g(on)g(a)o(v)o(erage,)g(su\016cient)g(bu\013ering)h(for)f(512)0 1931 y (cells,)21 b(but)d(is)h(capable)h(of)e(holding)i(up)f(to)f(1280)f(cells)j(dur\ ing)f(p)r(eak)f(tra\016c)g(p)r(erio)q(ds.)30 b(Switch)18 b(delay)g(is)h(less) 0 1987 y(than)c(10)g(microseconds)h(if)f(the)h(output)e(FIF)o(O)h(is)h(empt)o (y)l(.)0 2153 y fh(3.)52 b(The)18 b(problem)f(with)h(la)o(y)o(ered)f(proto)q (cols)h(o)o(v)n(er)f(A)-5 b(TM)0 2296 y fj(A)18 b(simple)i(w)o(a)n(y)f(to)e (incorp)s(orate)g(a)h(new)h(net)o(w)n(ork)f(tec)o(hnology)h(suc)o(h)f(as)g (A)m(TM)g(in)o(to)g(an)g(existing)i(op)q(erating)0 2353 y(system)c(is)h(to)f (la)o(y)o(er)g(it)h(b)r(eneath)f(an)h(existing)g(proto)q(col)g(stac)n(k)g(suc) o(h)g(as)f(TCP/IP)g(or)g(UDP/IP)l(.)g(In)h(this)g(w)o(a)n(y)l(,)0 2409 y(A)l (TM)d(pro)o(vides)h(the)f(physical)h(transp)q(ort)f(only)l(,)h(analogous)f (to)f(an)i(Ethernet,)f(a)g(tok)n(en)h(ring,)f(or)g(ev)o(en)h(a)f(serial)0 2466 y(line.)34 b(A)19 b(lay)n(ered)h(implemen)q(tation)f(w)n(ould)h(ha)o(v)o(e)f (the)h(A)l(TM)f(la)o(y)o(er)g(mo)n(v)o(e)h(cells)h(through)e(to)f(the)i(hardw) n(are)0 2514 y 780 2 v 52 2571 a fc(1)69 2587 y fi(The)13 b(F)m(ore)g(interfa\ ce)g(deliv)q(ers)g(sligh)q(tly)g(less)h(bandwidth)h(than)f(the)f(155)g(Mbps)h (capacit)q(y)f(of)f(the)i(net)o(w)n(ork.)52 2647 y fc(2)69 2663 y fi(The)j (n)o(um)o(b)q(er)g(of)f(cycles)i(required)g(dep)q(ends)g(on)e(the)h(particula\ r)i(TURBOc)o(hannel)f(implemen)q(tation;)h(the)e(minimum)g(is)g(4)0 2708 y (cycles)d([9].)964 2844 y fj(3)p eop %%Page: 4 7 bop 0 42 a fj(FIF)n(Os.)38 b(The)21 b(next)g(la)o(y)n(er)g(w)o(ould)g(p)r (erform)f(SAR,)h(p)r(ossibly)h(using)f(link)q(ed)g(bu\013ers)g(since)h(the)f (SAR)g(la)o(y)o(er)0 98 y(do)q(es)d(not)f(kno)o(w)g(the)h(total)f(message)g (length)h(and)g(cannot)f(allo)r(cate)h(a)f(con)o(tiguous)h(bu\013er)f(of)g (the)h(total)f(size)0 154 y(necessary)j(on)g(reception.)36 b(A)20 b(CS)g(la)o (y)o(er)g(on)g(top)g(of)f(SAR)i(w)o(ould)g(insert)f(or)g(remo)n(v)o(e)g(the)g (CS)g(header)h(and)0 211 y(trailer,)d(in)h(the)e(latter)h(case)f(v)o(erifying) i(that)d(the)i(reassem)n(bled)h(length)g(equals)f(that)f(in)h(the)g(CS)f(trai\ ler.)28 b(IP)l(,)0 267 y(which)13 b(pro)o(vides)h(routing,)g(fragmen)n(tation) f(and)h(reassem)o(bly)m(,)f(w)o(ould)h(b)r(e)f(la)o(y)o(ered)h(on)g(top)f(of) g(CS.)g(UDP)l(,)g(which)0 324 y(pro)o(vides)k(connectionless)i(service)f(and) f(dem)o(ultiplexing)j(of)d(pac)o(k)n(ets)g(to)f(user)h(endp)s(oin)o(ts,)g(and) g(TCP)l(,)f(which)0 380 y(pro)o(vides)e(connection-oriented,)f(reliable)i(str\ eam)e(comm)n(unication)h(with)g(automatic)e(\015o)o(w)h(con)o(trol)f(and)i (error)0 437 y(reco)o(v)n(ery)m(,)k(w)o(ould)h(in)g(turn)f(b)r(e)g(la)o(y)o (ered)g(on)h(top)e(of)h(IP)m(.)g(Finally)m(,)h(so)q(c)o(k)o(ets,)f(which)g (pro)o(vide)h(application-lev)q(el)0 493 y(endp)r(oints)14 b(and)i(an)f(addit\ ional)h(bu\013ering)g(la)o(y)o(er)f([19)o(],)g(w)n(ould)h(b)r(e)f(placed)i (on)e(top)g(of)f(UDP)h(and)h(TCP)-5 b(.)62 591 y(Sev)o(eral)17 b(existing)h (implementations)e(follo)o(w)h(this)f(approac)o(h)g([29,)f(3].)23 b(While)18 b (a)e(la)o(y)o(ered)h(implementation)0 648 y(reuses)h(existing)h(proto)q(col)f (soft)n(w)o(are,)f(it)i(also)f(in)o(tro)q(duces)g(redundancies)i(that)d(a\013\ ect)h(end-to-end)h(latencies)0 704 y(and)c(bandwidth)i([33)o(,)e(17)o(].)68 851 y ff(\017)23 b fj(Sp)q(ecial)17 b(connection)f(managemen)o(t)f(and)g(rout\ ing)h(mec)o(hanisms)g(are)f(required)h(b)q(ecause)g(A)l(TM)f(uses)h(its)114 908 y(o)n(wn)e(signaling)h(protocols)f(to)f(establish)h(and)g(tear)f(do)o(wn) g(connections.)20 b(These)14 b(proto)q(cols)g(are)f(di\013eren)o(t)114 964 y (from)h(the)h(ones)h(used)f(b)o(y)h(higher-level)g(proto)q(cols)f(suc)o(h)h (as)f(TCP)-5 b(.)68 1058 y ff(\017)23 b fj(The)17 b(A)m(TM)g(SAR)h(protocol)h (mak)n(es)e(IP)h(fragmen)o(tation)f(and)g(reassem)o(bly)h(redundant.)27 b(Mul\ tiple)19 b(seg-)114 1114 y(men)o(tation)c(and)g(reassem)n(bly)h(lay)n(ers)f (increase)i(latency)e(b)r(ecause)h(a)f(message)f(m)o(ust)h(b)r(e)g(brok)n(en) h(up)g(and)114 1171 y(reconstructed)g(m)o(ultiple)i(times,)f(and)g(decrease)g (usable)g(bandwidth)h(b)q(ecause)f(SAR)g(structural)g(infor-)114 1227 y(matio\ n)k(m)o(ust)g(b)q(e)h(carried)g(in)g(the)g(message.)38 b(IP)22 b(t)o(ypically) h(enforces)f(a)f(maxim)o(um)g(pac)o(k)o(et)g(length)114 1284 y(\(MTU\))16 b (of)h(576)f(b)o(ytes)518 1267 y fd(3)538 1284 y fj(.)26 b(An)o(y)17 b(pac)o (k)o(ets)g(larger)g(than)g(that)g(are)g(fragmen)n(ted)h(and)f(reassem)n(bled,) i(as)e(in)114 1340 y(A)l(TM)12 b(SAR.)h(T)l(o)f(prev)o(en)o(t)g(IP)h(fragmen) n(tation,)f(TCP)g(t)o(ypically)i(uses)f(a)f(maxim)o(um)g(segmen)o(t)g(size)h (\(MSS\))114 1396 y(of)k(512)f(or)h(536)f(b)o(ytes)h(\(the)h(slac)o(k)f(is)h (for)f(the)g(TCP/IP)g(header\).)27 b(As)17 b(IP)h(headers)f(alone)h(are)f(at) g(least)114 1453 y(20)g(b)o(ytes)g(long,)h(it)g(is)g(impractical)h(for)e(IP)h (to)f(use)h(an)g(MTU)f(of)g(44)g(b)o(ytes)h(\(the)f(maxim)o(um)h(AAL)g(3/4) 114 1509 y(segmen)o(t)12 b(length\).)19 b(The)13 b(TCP)f(header)h(adds)f(anot\ her)g(20)g(b)o(ytes,)h(so)f(it)h(is)g(imp)r(ossible)g(for)f(TCP)g(to)g(a)o (v)n(oid)114 1566 y(A)l(TM)g(SAR)h(b)q(ecause)g(ev)o(en)f(the)h(TCP/IP)e(head\ er)i(alone)g(cannot)f(b)q(e)h(transmitted)f(in)h(a)f(single)h(AAL)g(3/4)114 1622 y(cell)k(\(a)f(single-segment)g(AAL)g(3/4)g(message)f(can)h(hold)h(only) g(36)e(b)o(ytes\).)22 b(The)16 b(A)m(TM)f(interface)h(w)n(ould)114 1679 y(th) n(us)i(need)g(to)f(inform)h(IP)g(of)f(a)g(larger,)h(fak)n(e)g(MTU,)f(for)g (example)h(576)f(b)o(ytes,)g(and)h(then)g(do)g(its)f(o)o(wn)114 1735 y(SAR)e (to)f(break)h(the)f(large)h(IP)g(fragmen)n(ts)g(in)o(to)f(44-b)o(yte)g(segmen) o(ts.)19 b(Consequently)l(,)c(the)f(message)h(ma)n(y)114 1792 y(b)q(e)i(fragm\ en)n(ted)g(m)o(ultiple)h(times,)f(once)g(b)o(y)g(IP)g(\(or)f(prev)o(en)o(tiv) o(ely)i(b)o(y)e(TCP\),)g(and)h(then)g(again)f(b)o(y)h(the)114 1848 y(A)l(TM)e (SAR.)h(While)h(an)f(outb)q(oard)f(pro)q(cessor)g(can)h(p)r(erform)e(A)m(TM)h (SAR)h(pro)q(cessing)h(to)e(reduce)h(CPU)114 1905 y(o)n(v)o(erhead,)e(it)g (do)q(es)g(not)f(eliminate)j(the)e(latency)g(and)g(bandwidth)h(costs)e(of)g (redundant)g(fragmen)o(tation.)68 1998 y ff(\017)23 b fj(The)c(small)h(A)l (TM)e(pac)o(k)o(et)h(size)h(leads)f(to)f(p)r(o)q(or)g(net)o(w)o(ork)g(e\016ci\ ency)j(when)e(used)h(to)e(carry)g(IP)i(tra\016c)114 2055 y(b)q(ecause)c(IP)g (headers)g(are)g(relativ)o(ely)h(large)e(compared)h(to)f(the)h(size)h(of)e (application)i(data.)k(The)16 b(distri-)114 2111 y(bution)h(of)f(TCP)g(pac)o (k)o(et)g(size)h(for)f(application)j(data)c(is)i(sharply)g(bi-mo)r(dal,)g(wit\ h)g(a)f(ma)s(jor)f(p)r(eak)h(at)g(0)114 2168 y(b)o(ytes)g(and)h(a)f(secondary) g(p)r(eak)g(at)g(the)h(MSS)g([2)o(].)23 b(The)17 b(same)f(measuremen)o(ts)g (ha)o(v)o(e)h(sho)n(wn)g(that)f(77%)114 2224 y(of)e(all)j(TCP)e(pac)n(k)o(ets) g(carry)g(b)r(et)n(w)o(een)h(0)f(and)g(36)g(b)o(ytes)g(of)g(application)i(dat\ a;)d(6%)h(b)q(et)o(w)n(een)h(37)f(and)h(80)114 2280 y(b)o(ytes;)j(2%)f(b)q (et)o(w)n(een)h(81)f(and)g(124)g(b)o(ytes,)g(10%)g(carry)f(512)h(b)o(ytes,)g (and)h(4%)f(carry)f(536)h(b)o(ytes.)29 b(That)114 2337 y(data)15 b(could)j (b)q(e)f(transmitted)f(without)g(TCP/IP)g(headers)g(in)i(1,)d(2,)h(3,)g(12)g (or)g(13)g(AAL)g(3/4)g(segmen)o(ts,)114 2393 y(resp)q(ectively)m(,)e(requirin\ g)h(an)g(a)n(v)o(erage)f(of)g(2.7)f(cells)j(p)r(er)e(message.)19 b(The)c(TCP/\ IP)f(header)h(adds)f(at)g(least)114 2450 y(40)k(b)o(ytes)h(to)g(the)g(applica\ tion)i(data)e(and)h(raises)f(the)h(a)n(v)o(erage)f(n)o(um)n(b)r(er)g(of)g(cel\ ls)i(p)r(er)e(pac)o(k)n(et)h(to)e(3.7.)114 2506 y(This)c(decreases)g(the)g (transmission)h(e\016ciency)g(b)o(y)f(37%.)19 b(Similar)c(observ)m(ations)f (ab)q(out)g(UDP)f(b)r(eha)o(vior)114 2563 y(rev)n(eal)k(that)f(headers)g(rais\ e)h(the)f(a)o(v)n(erage)g(n)o(um)o(b)r(er)g(of)g(cells)i(from)d(2.1)g(to)h (2.5,)f(which)h(is)h(a)f(19%)g(loss)g(in)114 2619 y(e\016ciency)m(.)0 2667 y 780 2 v 52 2724 a fc(3)69 2740 y fi(Larger)d(MTUs)g(can)g(b)r(e)g(used)g(in)h (sp)r(eci\014c)g(LANs)964 2844 y fj(4)p eop %%Page: 5 8 bop 68 42 a ff(\017)23 b fj(TCP's)13 b(\015o)o(w)h(con)o(trol)g(sc)o(heme)h (reacts)f(to)g(congestion)g(b)o(y)h(reducing)g(window)f(sizes)h(and)g(b)o(y)f (retransmit-)114 98 y(ting)f(lost)f(segmen)o(ts)h([24)o(],)f(but)h(also)g(rel\ ies)h(on)f(implicit)i(lo)o(w)o(er-lev)o(el)f(limits)g(that)e(do)h(not)f(exist) h(naturally)114 154 y(in)19 b(A)m(TM)f(net)o(w)n(orks.)30 b(F)l(or)19 b(examp\ le,)h(on)e(Ethernet)h(or)f(FDDI)h([26)o(],)g(only)g(one)g(host)f(or)h(gatew)n (a)n(y)g(can)114 211 y(transmit)12 b(at)h(a)f(time,)i(so)f(an)n(y)g(attac)n (hed)h(no)q(de)g(can)f(nev)o(er)g(receive)g(data)f(ab)r(o)n(v)o(e)h(the)g(\ \014xed)h(net)o(w)n(ork)f(rate.)114 267 y(In)f(an)f(A)m(TM)g(net)o(w)n(ork,)h (though,)f(switc)o(hes)h(can)g(receive)g(data)f(at)f(m)o(ultiple)k(input)f (p)q(orts)e(sim)o(ultaneously)m(,)114 324 y(and)17 b(at)g(eac)o(h)g(p)r(ort)f (at)h(full)h(link)h(bandwidth.)27 b(If)18 b(all)g(data)e(is)i(destined)h(to)d (a)h(single)i(output)e(p)q(ort)g(and)114 380 y(a)n(v)n(ailable)h(bu\013ering) f(is)g(\014nite,)h(data)e(loss)g(will)j(result.)24 b(Moreo)n(v)o(er,)16 b(TCP\ 's)g fg(slow)h(start)g fj(mec)o(hanism)g([6)o(])114 437 y fg(c)m(auses)e fj (data)g(loss,)h(ev)n(en)g(in)h(the)e(absence)h(of)f(cross)g(tra\016c)g(\(tra\ \016c)g(from)f(other)h(connections)i(tra)n(v)o(eling)114 493 y(o)n(v)o(er)e (the)g(same)g(path)f(or)h(subpath\).)20 b(Slo)o(w)15 b(start)f(increases)h (the)h(windo)o(w)f(size)h(eac)o(h)f(time)g(an)g(ac)o(kno)n(wl-)114 550 y(edge\ men)o(t)j(is)h(received.)29 b(Ev)o(en)o(tually)m(,)19 b(the)f(bu\013ering)h (capacit)o(y)f(of)g(the)g(destination)h(or)f(intermediate)114 606 y(no)q(des) e(is)g(exceeded,)h(causing)g(pac)o(k)o(et)e(loss,)h(and)g(reducing)h(the)f (window)f(bac)o(k)h(to)f(one)h(segmen)o(t.)21 b(As)16 b(a)114 663 y(result,)f (windo)o(w)g(sizes)h(oscillate)h(b)q(et)o(w)o(een)e(extremes)g([28)o(].)62 805 y(Finally)n(,)h(in)g(a)g(la)o(y)n(ered)h(implementation)f(suc)o(h)g(as)f(the) h(one)g(outlined)h(ab)r(o)n(v)o(e,)f(a)f(message's)g(data)g(m)o(ust)g(b)r(e)0 861 y(manipulated)d(and)g(copied)g(m)o(ultiple)h(times:)18 b(\014rst)11 b(fro\ m)f(the)h(hardw)o(are)f(FIF)o(Os)h(to)f(cells)j(in)f(host)e(memory)l(,)i(then) 0 918 y(from)f(cells)i(into)e(linked)i(bu\013ers)e(\(SAR\),)h(then)g(into)f (pro)q(cessor)g(registers)h(for)f(c)o(hec)o(ksumming)i(\(TCP)e(or)g(UDP\),)0 974 y(and)k(\014nally)i(into)d(user)i(bu\013ers)f(\(so)q(c)o(k)n(et)g(la)o (y)o(er\).)20 b(The)15 b(transmit)g(path)g(is)h(symmetric.)k(This)c(b)r(eha)o (vior)f(w)o(astes)0 1031 y(CPU)e(cycles)i(and)f(in)o(teracts)f(badly)h(with)g (the)g(limited)h(memory)e(and)h(I/O)f(bus)h(bandwidth)h(commonly)f(found)0 1087 y(in)g(w)o(orkstations)e([23)o(].)19 b(F)l(or)13 b(example,)h(memory-to-\ memory)e(cop)o(y)i(bandwidth)g(on)g(a)f(DECstation)f(5000/200)0 1144 y(is)j (limited)i(to)d(ab)q(out)h(160)e(Mbps,)i(which)g(is)g(only)g(slightly)g(great\ er)f(than)h(the)f(155)g(Mbps)h(A)l(TM)g(transmission)0 1200 y(rate,)j(and)h (substan)o(tially)h(less)f(than)f(the)h(622)f(Mbps)g(transmission)h(rate)f (in)h(future)g(A)l(TM)f(net)o(w)o(orks.)29 b(An)o(y)0 1257 y(protocol)16 b (la)o(y)o(er)f(or)f(subla)o(y)o(er)h(that)f(references)i(the)f(data)f(in)i (the)f(pac)o(k)n(et)g(will)i(tak)n(e)e(time)g(comparable)g(to)g(that)0 1313 y (necessary)g(to)e(transfer)h(the)h(data)e(o)o(v)o(er)h(the)h(ph)o(ysical)g (net)o(w)o(ork.)k(With)c(m)n(ultiple)i(protocol)e(la)o(y)o(ers)f(and)h(passes) 0 1370 y(o)n(v)o(er)k(the)g(same)f(data,)h(proto)q(col)g(pro)q(cessing)h(quic\ kly)f(b)q(ecomes)g(the)g(b)r(ottlenec)o(k)g(in)h(the)f(pip)r(eline)i(b)q(et)o (w)n(een)0 1426 y(comm)n(unicating)c(applications)g(on)e(di\013eren)o(t)g(hos\ ts.)0 1592 y fh(4.)52 b(An)17 b(in)o(tegrated)h(proto)r(col)g(arc)o(hitecture) 0 1735 y fj(T)l(o)12 b(address)h(the)g(problems)g(of)f(redundant)g(pro)q(cess\ ing)h(outlined)h(in)g(the)e(previous)i(section,)f(w)o(e)f(ha)o(v)o(e)h(design\ ed)0 1791 y(and)k(implemented)g(a)f(protocol)h(arc)o(hitecture)g(sp)q(eci\014\ cally)i(tailored)e(to)f(the)g(requirements)g(and)h(capabilities)0 1848 y(of)j (A)m(TM.)g(As)h(previously)h(men)o(tioned,)h(w)n(e)e(are)g(fo)q(cusing)h(on)f (the)g(case)g(of)f(lo)r(cal)h(and)g(metrop)q(olitan)h(area)0 1904 y(net)o(w)n (orking,)14 b(sp)r(eci\014cally)i(o)n(v)o(er)e(distances)h(limited)g(to)e(ab) r(out)g(40)g(km.)20 b(Additionally)n(,)14 b(in)h(the)f(w)n(ork)f(presented)0 1961 y(in)19 b(this)g(pap)r(er)f(w)o(e)g(are)h(concerned)g(exclusiv)q(ely)g (with)g(data)f(comm)n(unication,)i(supp)r(orting)f(only)g(statistical)0 2017 y (m)o(ultiplexing)f(of)c(net)o(w)o(ork)h(bandwidth.)62 2115 y(Our)21 b(net)n (w)o(ork)e(arc)o(hitecture)i(supp)r(orts)e(comm)n(unication)i(b)r(et)o(w)n (een)g fg(endp)m(oints)p fj(,)g(whic)o(h)f(are)g(the)g(en)o(tities)0 2171 y (through)h(whic)o(h)h(user)f(tasks)f(send)i(or)f(receive)g(messages.)37 b(An) 22 b(endp)r(oin)o(t)f(is)h(analogous)e(to)h(a)g(BSD)g(Unix)0 2228 y(so)q(c)o (k)o(et)15 b([19)o(].)20 b(The)15 b(task)g(that)g(creates)g(an)g(endp)r(oin)o (t)g(is)h(its)g fg(owner)p fj(,)f(and)g(is)h(the)g(only)g(one)f(allow)n(ed)h (to)f(send)g(or)0 2284 y(receive)g(messages)g(through)g(it.)62 2382 y(Endp)r (oints)e(ha)n(v)o(e)h(associated)h fg(bu\013er)g(ar)n(e)m(as)f fj(that)e(are) i(co-mapp)q(ed)g(into)e(the)i(address)g(spaces)f(of)h(the)f(k)o(ernel)0 2439 y (and)k(the)h(endp)r(oin)o(t)f(o)o(wner)g(task.)25 b(Bu\013ers)17 b(for)g(rece\ iv)o(ed)h(messages)f(are)g(allo)r(cated)g(from)f(the)i(bu\013er)f(area)f(of)0 2495 y(the)g(receiving)h(endp)s(oin)o(t.)22 b(An)16 b(endp)r(oin)o(t's)f(prot\ o)q(col)h(and)g(bu\013er)g(area)f(size)i(are)f(de\014ned)h(when)f(it)h(is)f (created)0 2552 y(b)o(y)h(a)g(task.)26 b(The)18 b(protocol)g(determines)h(the) e(transp)q(ort)g(discipline)k(applied)e(to)e(messages)g(sen)o(t)g(or)g(receiv\ ed)0 2608 y(through)e(the)g(endp)r(oint,)f(and)i(can)f(b)q(e)h(one)f(of)g(dat\ agrams,)e(sequenced)k(pac)o(k)n(ets)e(and)h(ra)n(w)f(datagrams.)68 2740 y ff (\017)23 b fj(Datagrams)17 b(are)j(sen)o(t)f(o)n(v)o(er)h(a)f(m)o(ultiplexed) j(connection)f(b)q(et)o(w)n(een)g(the)f(comm)n(unicating)h(hosts,)f(and)964 2844 y(5)p eop %%Page: 6 9 bop 114 42 a fj(are)13 b(automatically)i(\015o)n(w)f(con)o(trolled.)20 b(A) 14 b(datagram)f(endp)r(oin)o(t)h(can)g(sim)o(ultaneously)i(receiv)o(e)e(messa\ ges)114 98 y(from)g(m)o(ultiple)j(hosts.)68 189 y ff(\017)23 b fj(Sequenced) 17 b(pac)o(k)n(ets)f(are)f(transp)q(orted)g(o)n(v)o(er)g(an)h(exclusive)g(con\ nection)g(b)r(et)o(w)n(een)g(the)g(resp)q(ective)f(end-)114 245 y(p)q(oin)o (ts,)f(and)g(are)g(automatically)g(n)o(um)o(b)r(ered,)g(ac)n(kno)o(wledged,)h (and)f(retransmitted)g(in)h(case)f(of)f(error)g(or)114 302 y(time-out.)68 393 y ff(\017)23 b fj(Ra)o(w)12 b(datagrams)g(are)h(pro)n(vided)i(for)d(internet) n(w)o(orking)h(and)g(wide-area)h(accesses.)19 b(Ra)o(w)13 b(datagrams)e(un-) 114 449 y(dergo)h(only)i(A)l(TM,)e(SAR)i(and)f(CS)g(la)o(y)o(er)g(pro)q(cessi\ ng.)20 b(They)13 b(are)g(neither)h(reliable)h(nor)e(\015o)o(w-con)n(trolled.) 114 506 y(These)19 b(functions)h(are)g(left)f(to)g(higher-lev)q(el)h(soft)n (w)n(are.)32 b(Like)20 b(datagrams,)e(ra)n(w)h(datagrams)f(are)h(sen)o(t)114 562 y(o)n(v)o(er)c(m)o(ultiplexed)i(connections.)62 690 y(Datagrams)d(and)i (sequenced)h(pac)o(k)n(ets)f(can)g(b)q(e)g(up)g(to)f(64)g(KB)h(long,)g(and)g (are)f(intended)i(for)e(the)g(common)0 747 y(case)k(of)g(hosts)f(connected)j (b)o(y)e(A)l(TM)f(in)i(the)g(same)f(lo)q(cal)h(or)e(metrop)q(olitan)i(area)f (net)n(w)o(ork.)31 b(W)l(e)19 b(limit)i(the)0 803 y(size)15 b(of)f(ra)n(w)h (datagrams)d(to)i(9244)f(b)o(ytes)h(b)r(ecause)h(they)f(are)g(not)g(\015o)o (w-con)o(trolled)h(\(this)g(limit)g(w)o(as)f(c)o(hosen)h(for)0 860 y(compatib\ ilit)q(y)j(with)i(IEEE)f(802.6)e(and)i(with)g(the)g(size)h(of)e(the)h(receive) g(FIF)n(Os)g(of)g(the)g(hardw)n(are)g(w)n(e)g(use\).)0 916 y(The)c(proto)q (col)h(of)e(comm)o(unicating)i(endp)s(oin)o(ts)f(m)n(ust)g(b)r(e)g(compatible\ .)62 1014 y(Our)f(protocols)g(are)f(implemented)g(within)i(a)d(k)o(ernel-lev) q(el)i(device)g(driver)f(that)f(pro)o(vides)i(the)f(A)l(TM,)g(SAR,)0 1071 y (CS)21 b(and)h(signaling)h(la)o(y)o(ers,)f(as)f(w)n(ell)i(as)e(the)h(\015o)n (w)f(con)o(trol)h(and)f(error)g(reco)n(v)o(ery)g(required)i(for)e(datagrams)0 1127 y(and)16 b(sequenced)i(pac)o(k)o(ets.)k(The)17 b(driv)o(er)g(is)f(organi\ zed)h(in)g(a)f(single)h(lay)n(er)f(and)h(runs)f(entirely)h(in)g(k)n(ernel)h (mo)q(de)0 1184 y(for)e(protection.)26 b(Other)17 b(proto)q(cols,)g(suc)o(h)h (as)e(TCP/IP)l(,)h(or)f(application)j(programming)e(in)o(terfaces,)g(suc)o (h)g(as)0 1240 y(so)q(c)o(k)o(ets,)d(can)h(b)r(e)g(la)o(y)o(ered)g(on)g(top)g (of)f(the)i(nativ)o(e)f(in)o(terface)g(for)f(compatibilit)q(y)l(.)20 b(These) c(higher)f(la)o(y)o(ers)g(can)g(b)r(e)0 1297 y(implemen)q(ted)g(inside)i(the) e(k)o(ernel)h(or)f(at)f(user)i(lev)o(el)g([20)o(].)62 1394 y(In)j(the)f(rest) f(of)g(this)h(section)h(w)n(e)f(discuss)h(in)g(more)e(detail)i(our)e(approac) o(h)h(to)f(bu\013er)h(managemen)n(t,)g(seg-)0 1451 y(men)o(tation)h(and)h(rea\ ssem)o(bly)m(,)g(\015o)o(w)f(con)o(trol,)h(error)f(con)o(trol)g(and)h(reco)o (v)o(ery)l(,)g(signaling,)i(m)o(ultiplexing,)h(and)0 1507 y(integration)14 b (with)i(In)o(ternet)f(proto)q(cols.)0 1668 y fh(4.1.)52 b(Bu\013er)16 b(manag\ emen)o(t)0 1795 y fj(Protocol)j(implemen)q(tations)g(often)f(use)h(their)h (o)n(wn)f(bu\013ers,)g(distinct)h(from)e(the)h(bu\013ers)g(used)h(b)o(y)e(the) h(end)0 1851 y(applications.)30 b(In)19 b(BSD)f(Unix,)h(for)f(example,)h(prot\ ocols)g(use)f fg(mbufs)g fj([19)o(],)g(while)i(applications)g(use)e(bu\013ers) 0 1908 y(allo)r(cated)d(from)f(the)h(heap)g(or)f(stac)n(k.)20 b(This)c(imp)r (oses)e(at)g(least)h(one)g(extra)f(data)g(cop)o(y)h(through)f(m)o(bufs)h(on)g (the)0 1964 y(path)g(b)r(et)n(w)o(een)h(the)f(application)i(and)e(the)g(net)o (w)o(ork.)62 2062 y(W)l(e)20 b(integrate)f(bu\013er)h(managemen)n(t)g(for)f (protocols)i(and)f(applications,)i(and)e(therefore)g(eliminate)i(this)0 2119 y (cop)o(y)l(.)31 b(Endp)r(oin)o(ts)18 b(ha)o(v)o(e)h(exclusive)g(bu\013er)g (areas)f(that)g(are)h(statically)g(mapp)r(ed)g(b)q(et)o(w)n(een)h(the)f(k)n (ernel-lev)q(el)0 2175 y(protocol)c(driv)o(er)f(and)g(the)f(endp)s(oin)o(t's) g(associated)i(address)e(space.)20 b(When)14 b(a)f(message)h(arriv)n(es)g(fro\ m)f(the)h(net-)0 2232 y(w)o(ork,)e(the)g(net)o(w)o(ork)g(interface)g(allo)r (cates)g(the)h(bu\013er)f(for)g(it)h(from)f(the)h(bu\013er)f(area)g(of)g(the) h(receiving)h(endp)r(oint.)0 2288 y(Since)20 b(this)g(bu\013er)f(area)f(is)i (already)f(mapp)q(ed)h(to)e(the)h(endp)r(oint's)f(o)n(wner)h(address)g(space,) h(the)f(application)0 2345 y(can)c(receive)g(the)g(message)f(with)h(no)g(copi\ es)h(or)e(virtual)h(memory)g(manipulations.)21 b(When)15 b(sending)h(data,)e (the)0 2401 y(sender)20 b(blo)r(c)o(ks)g(un)o(til)g(the)g(message)f(has)h(b)q (een)g(sent)f(\(and,)h(in)h(the)e(case)h(of)f(sequenced)i(pac)o(k)o(ets,)f (ac)o(kno)n(wl-)0 2457 y(edged\).)31 b(This)19 b(a)n(v)o(oids)g(ha)o(ving)g (to)f(cop)o(y)h(an)g(outgoing)f(message)g(into)g(an)h(in)o(termediate)g(bu\ \013er.)31 b(Outgoing)0 2514 y(messages)16 b(should)h(b)q(e)g(allo)r(cated)f (from)g(the)g(mapp)q(ed)h(bu\013ers)f(to)g(ensure)g(that)g(the)g(driver)g(alw) o(a)n(ys)g(has)g(rapid)0 2570 y(access)k(to)e(message)h(data,)h(ev)n(en)g(whe\ n)g(the)g(curren)o(tly)f(running)i(address)e(space)h(do)q(es)g(not)f(o)n(wn)h (the)f(data.)0 2627 y(This)e(is)g(necessary)f(during)h(the)g(transmission)g (of)e(a)h(long)h(message)f(that)g(ma)n(y)g(span)h(address)f(space)h(con)n(tex\ t)0 2683 y(switc)o(hes.)33 b(The)20 b(bu\013ering)g(structure)f(is)h fg(exp)n (ose)n(d)f fj(and)h(applications)h(are)e(co-resp)r(onsible)i(for)e(ho)o(w)g (bu\013ers)0 2740 y(are)f(utilized.)30 b(If)18 b(m)n(ultiple)i(threads)e(run) g(in)h(the)f(address)g(space)g(from)f(which)h(a)f(message)g(is)i(sen)o(t,)f (it)g(it)g(the)964 2844 y(6)p eop %%Page: 7 10 bop 0 42 a fj(application's)16 b(resp)q(onsibilit)q(y)f(to)f(sync)o(hronize) h(bu\013er)g(accesses)g(as)f(necessary)g(for)g(integrit)n(y)m(.)19 b(Bu\013er\ s)c(can)f(b)r(e)0 98 y(deallo)r(cated)i(either)g(immediately)g(after)f(b)q (eing)h(sen)o(t,)f(or)g(later)g(b)o(y)g(an)g(explicit)i(call.)62 196 y(Integr\ ated)12 b(bu\013er)h(managemen)n(t)g(do)q(es)g(not)f(cause)h(memory)g(protect\ ion)g(problems)g(b)r(ecause)g(the)g(net)o(w)n(ork)0 252 y(interface)i(runs)h (inside)i(the)e(k)o(ernel)h(and)f(is)h(trusted,)f(and)g(eac)o(h)g(application) h(can)g(only)f(access)g(the)g(bu\013ers)g(of)0 309 y(the)e(endp)r(oin)o(ts)f (it)h(o)n(wns.)20 b(Misb)r(eha)n(vior)14 b(b)o(y)g(an)f(application,)i(e.g.)k (not)13 b(freeing)h(message)f(bu\013ers,)h(only)g(a\013ects)0 365 y(that)g (application,)j(since)f(the)g(bu\013er)f(areas)f(of)h(eac)o(h)g(endp)s(oin)o (t)g(are)g(separate.)0 527 y fh(4.2.)52 b(Segmen)n(tation)19 b(and)f(reassem) m(bly)0 654 y fj(Because)e(of)e(A)l(TM's)g(extremely)i(small)g(cell)g(size,)g (segmen)o(tation)e(and)i(reassem)n(bly)g(p)q(erformance)f(are)g(critical)0 711 y(to)j(o)o(v)n(erall)i(net)o(w)o(ork)e(p)q(erformance.)31 b(T)l(o)19 b(minimi\ ze)i(the)e(impact)g(of)f(SAR,)h(w)o(e)g(reduce)h(the)f(n)o(um)o(b)q(er)g(of)f (cell)0 767 y(manipulation)g(steps)e(b)o(y)g(com)n(bining)i(the)e(A)m(TM,)f (SAR)i(and)f(CS)g(functionality)g(into)f(a)h(single)i(pass)e(o)n(v)o(er)g(the) 0 824 y(data.)62 922 y(The)d(A)l(TM,)e(SAR,)i(and)f(CS)g(headers,)h(trailers,) g(and)f(pads)h(are)e(generated)i(or)e(v)o(eri\014ed)i(on)g(the)f(\015y)g(as)g (data)f(is)0 978 y(transferred)g(into)g(or)g(out)g(of)g(the)g(FIF)o(Os.)19 b (They)11 b(are)h(nev)o(er)f(stored)g(in)i(memory)e(during)h(either)g(send)g (or)f(receive.)0 1034 y(When)h(the)h(b)q(eginning)h(segmen)o(t)d(of)h(a)g(mes\ sage)f(is)i(received,)f(the)h(CS)f(header)g(is)h(read)f(to)f(determine)i(the) f(size)h(of)0 1091 y(the)f(incoming)g(message,)g(and)g(a)f(con)o(tiguous)g (bu\013er)h(of)f(the)g(appropriate)h(size)g(is)g(allo)r(cated)g(from)f(the)g (receiving)0 1147 y(endp)r(oint's)i(bu\013er)i(area.)k(The)14 b(receiving)i (endp)s(oin)o(t)e(is)h(determined)h(b)o(y)e(the)h(A)l(TM)f(header's)g(VPI/V)o (CI)h(\014eld,)0 1204 y(which)g(w)o(e)g(use)g(to)g(index)h(into)e(a)h(table)h (that)e(main)o(tains)i(a)e(list)i(of)f(free)g(bu\013ers)g(for)g(the)g(resp)q (ective)g(endp)r(oint.)0 1260 y(Once)f(the)g(bu\013er)g(has)f(b)r(een)h(allo) r(cated)f(in)i(the)e(receiver's)g(address)h(space,)f(the)h(net)o(w)n(ork)f (interface)g(reads)h(data)0 1317 y(w)o(ords)h(from)f(the)i(receiv)o(e)g(FIF)n (O)g(and)g(places)g(them)f(directly)i(in)f(the)g(correct)e(lo)r(cation)i(in)g (the)f(bu\013er.)21 b(When)0 1373 y(an)c(end)g(of)g(message)f(is)i(receiv)o (ed,)g(the)f(message)f(data)h(is)g(in)h(the)f(bu\013er)g(ready)f(for)h(use)g (b)o(y)g(the)g(application.)0 1430 y(The)d(result)h(is)g(that)e(A)m(TM,)g(SAR) i(and)f(CS)h(pro)q(cessing)g(can)f(b)r(e)g(p)r(erformed)f(at)h(a)g(rate)g(clo\ se)g(to)g(that)g(at)f(which)0 1486 y(the)i(host)g(CPU)g(can)g(transfer)g(w)n (ords)g(across)g(the)g(I/O)h(bus.)0 1648 y fh(4.3.)52 b(Flo)o(w)17 b(con)o (trol)h(and)g(congestion)g(a)o(v)n(oidance)0 1775 y fj(Net)o(w)n(ork)j(\015o) o(w)g(con)o(trol)g(is)h(required)g(to)f(ensure)h(that)f(a)g(sending)h(host)f (do)q(es)h(not)f(o)n(v)o(errun)h(the)f(resources)0 1831 y(a)n(v)n(ailable)c (at)e(the)g(receiver.)20 b(Ov)o(errun)c(is)g(p)q(ossible)h(b)q(ecause)f(a)f (receiver)g(m)o(ust)g(react)g(to)g(and)g(pro)q(cess)h(data)e(as)0 1888 y(it)j (arriv)o(es,)g(whereas)g(a)f(sender)i(ma)n(y)f(carefully)h(stage)e(the)h(tran\ sfer)g(for)f(maxim)o(um)h(bandwidth.)26 b(Our)17 b(basic)0 1944 y(approac)n (h)i(to)f(\015o)o(w)g(con)o(trol)g(is)h(to)f(allo)o(w)g(the)h(receiver)f(a)g (similar)i(degree)f(of)f(con)o(trol)g(o)n(v)o(er)g(data)g(staging)g(so)0 2001 y(that)c(it)i(can)f(predict)h(and)g(reserv)n(e)g(resources)f(on)g(a)g(message) g(b)o(y)g(message)g(basis.)62 2099 y(Because)g(of)f(the)g(nature)g(of)f(net)o (w)o(ork)g(tra\016c,)g(which)h(consists)h(of)e(man)o(y)h(small)h(messages)e (and)h(a)g(few)g(large)0 2155 y(messages,)21 b(our)g(\015o)o(w)g(con)o(trol)f (strategy)g(separates)g(messages)h(in)o(to)g(t)n(w)n(o)g(classes,)i fg(short) e fj(and)g fg(long)p fj(.)36 b(Short)0 2212 y(messages,)13 b(which)h(can)g (b)q(e)g(transp)q(orted)f(in)i(one)f(or)f(t)n(w)o(o)g(A)m(TM)g(cells,)i(and)f (which)g(accoun)o(t)f(for)g(almost)g(80%)g(of)0 2268 y(In)o(ternet)h(tra\016c\ ,)g(are)g(the)g(common)g(case)g(for)g(which)g(w)o(e)g(optimize.)21 b(W)l(e)14 b(transp)q(ort)g(short)f(messages)h(without)0 2325 y(an)o(y)k(explicit)j(\015\ o)n(w)d(con)o(trol.)30 b(Instead,)19 b(w)n(e)g(dedicate)g(a)f(small)i(fractio\ n)e(of)g(a)g(receiver's)f(resources)i(to)f(short)0 2381 y(messages.)35 b(Sinc\ e)22 b(the)f(bandwidth)h(and)e(bu\013ering)i(requiremen)o(ts)f(of)f(short)g (messages)g(are)g(minimal,)j(the)0 2437 y(dedicated)17 b(allotmen)o(t)e(is)h (more)e(than)h(enough)h(in)g(most)e(practical)i(situations.)62 2535 y(Long)11 b(messages,)g(greater)f(than)h(t)n(w)o(o)g(segmen)o(ts,)g(require)g(explicit) i(\015o)o(w)e(con)o(trol)f(and)i(congestion)f(a)n(v)o(oidance)0 2592 y(mec)o (hanisms.)20 b(W)l(e)13 b(pro)o(vide)h(\015o)o(w)f(con)o(trol)g(for)g(long)g (messages)g(with)h(a)f fg(sliding)g(window)h fj(protocol)g([31)o(].)19 b(Befo\ re)0 2648 y(a)g(long)h(message)f(is)g(transmitted,)h(it)g(is)g(brok)n(en)g (into)e fg(windows)p fj(,)i(which)f(are)h(the)f(unit)h(of)f(sync)o(hronizatio\ n)0 2705 y(b)r(et)n(w)o(een)h(sender)g(and)f(receiver.)32 b(After)19 b(sendin\ g)i(a)e(windo)o(w,)h(the)g(sender)g(m)n(ust)g(receiv)o(e)g(a)f(corresp)q(ondi\ ng)964 2844 y(7)p eop %%Page: 8 11 bop 0 42 a fg(synchr)m(onization)16 b(c)n(el)r(l)f fj(b)q(efore)h(sending)g (the)g(next)g(windo)o(w)g(in)g(the)g(message.)k(W)l(e)c(use)g(a)f(window)g (size)i(of)e(one)0 98 y(segmen)o(t)k(for)f(the)h(initial)j(windo)o(w)d(in)h (a)f(message,)g(71)g(segmen)o(ts)g(\(explained)h(b)r(elo)o(w\))f(for)f(all)i (but)g(the)f(last)0 154 y(windo)o(w,)e(and)f(b)r(et)n(w)o(een)h(1)f(and)h(71) e(segmen)o(ts)h(for)g(the)h(last)f(windo)o(w.)24 b(The)17 b(initial)h(windo)o (w,)f(b)q(eing)g(small,)g(is)0 211 y(unlikely)g(to)d(o)o(v)n(er\015o)o(w)h (the)h(receiv)o(er's)f(FIF)o(O.)g(If)g(a)g(host)g(receives)g(a)g(new)h(initia\ l)h(window)e(while)h(it)g(is)g(already)0 267 y(receiving)23 b(t)n(w)o(o)e(lon\ g)g(messages,)h(then)g(the)f(receiving)i(host)e(queues)h(the)g(corresp)q(ondi\ ng)g(sync)o(hronization)0 324 y(message.)d(The)c(sync)o(hronization)g(is)g (sen)o(t)g(when)g(there)f(is)h(only)g(one)g(long)g(message)f(outstanding.)19 b (Using)d(this)0 380 y fg(bu\013er)h(sche)m(duling)e fj(tec)n(hnique,)i(the)e (receiver)g(substan)o(tially)h(reduces)g(the)f(c)o(hances)h(of)f(FIF)n(O)h (o)n(v)o(errun.)62 478 y(The)h(c)o(hoice)g(of)f(windo)o(w)g(size)h(is)g(a)f (compromise)h(b)q(et)o(w)n(een)g(the)g(con\015icting)g(requirements)f(of)g (bandwidth)0 535 y(e\016ciency)d(and)f(bu\013ering)g(limitations.)20 b(In)12 b (our)g(windo)o(wing)g(sc)o(heme,)g(only)g(one)g(window)f(of)g(eac)o(h)h(messa\ ge)f(can)0 591 y(b)r(e)j(in)h(transit)g(at)e(an)o(y)i(giv)o(en)g(time,)f(and) h(the)f(receiver)h(sends)g(a)f(sync)o(hronization)h(cell)h(when)f(it)g(pro)q (cesses)g(the)0 648 y(initial)f(segmen)n(t)d(in)i(a)e(windo)o(w.)19 b(Small) 12 b(windo)o(ws)g(are)f(therefore)g(less)h(likely)g(to)f(cause)g(congestion;) i(only)f(a)f(large)0 704 y(n)o(um)o(b)q(er)k(of)g(concurrent)f(messages)h(can) g(cause)h(bu\013er)f(o)n(v)o(errun.)20 b(On)c(the)f(other)g(hand,)g(large)g (windows)g(allo)o(w)0 760 y(b)r(etter)d(utilization)i(of)e(the)h(a)n(v)n(aila\ ble)h(bandwidth)g(b)q(ecause)f(a)f(large)h(windo)o(w)g(allo)o(ws)f(the)h(sync) o(hronization)g(cell)0 817 y(to)k(o)o(v)n(erlap)i(with)f(the)g(transmission)g (of)f(the)h(window.)27 b(When)18 b(the)g(current)f(windo)o(w)h(has)g(b)q(een) h(completely)0 873 y(sen)o(t,)11 b(its)h(sync)o(hronization)g(has)f(already)h (arriv)o(ed)f(at)g(the)h(sender,)g(and)f(the)h(sender)g(can)f(con)o(tin)o(ue) h(transmitting)0 930 y(without)17 b(delay)l(.)26 b(Ov)o(erlap)18 b(o)q(ccurs) g(if)g fb(W)6 b(S)19 b ff(\025)d fb(RT)6 b(T)17 b ff(\002)12 b fb(B)s(W)6 b fj (,)17 b(where)h fb(W)6 b(S)20 b fj(is)e(the)f(window)g(size,)h fb(RT)6 b(T)23 b fj(is)18 b(the)0 986 y(round-trip)g(time,)f(and)g fb(B)r(W)23 b fj(is)18 b (the)e(a)o(v)m(ailable)j(bandwidth.)25 b(The)17 b(maxim)o(um)g(reception)h (bandwidth)g(of)e(the)0 1043 y(net)o(w)n(ork)21 b(interface)g(is)h(ab)q(out)f (50)f(Mbps,)j(and)e(the)h(maxim)n(um)g(transmission)f(bandwidth)i(is)f(around) f(110)0 1099 y(Mbps.)e(The)14 b(round-trip)f(time,)h(assuming)f(that)g(the)g (propagation)g(dela)o(y)g(is)h(5)f(microseconds/km,)g(that)f(hosts)0 1156 y (are)k(separated)f(b)o(y)h fb(d)f fj(km)h(and)g fb(s)g fj(switc)o(hes,)g(and) g(that)f(ev)o(ery)h(queue)h(encoun)o(tered)f(on)g(the)g(path)g(is)g(empt)o (y)l(,)f(is)0 1212 y fb(RT)6 b(T)18 b fj(=)13 b(10)p fb(d)c fj(+)h(20)p fb (s)p fj(.)19 b(Switches)c(with)g(16)g(p)q(orts)f(can)h(handle)i(up)e(to)f(360\ 0)g(hosts)h(suc)o(h)g(that)f(no)h(t)o(w)n(o)g(hosts)f(are)0 1269 y(more)h(tha\ n)g(5)g(switc)o(hes)h(a)n(w)o(a)n(y)g(from)e(eac)o(h)h(other.)20 b(A)c(net)o (w)n(ork)f(of)g(switc)o(hes)h(with)f(64)g(p)q(orts)g(eac)o(h)g(can)h(handle)0 1325 y(o)n(v)o(er)f(250,000)e(hosts)h(with)h(a)g(maxim)o(um)g(distance)h(of)e (5)h(switc)o(hes.)20 b(Assuming)15 b(hosts)g(separated)f(b)o(y)h(up)g(to)g (40)0 1381 y(km)h(and)g(5)f(switc)o(hes,)h(a)g(windo)o(w)g(size)g(of)g(71)f (segmen)o(ts)g(enables)i(transmission)f(at)f(the)h(maxim)o(um)g(reception)0 1438 y(bandwidth.)62 1536 y(The)h(FIF)n(O)g(space)f(reserv)o(ed)h(for)f(recep\ tion)h(of)f(a)g(long)g(message)g(will)i(dep)r(end)f(on)f(the)h(actual)f fb (RT)6 b(T)g fj(,)16 b(and)0 1592 y(on)d(the)f(relative)h(send)g(and)g(receive) f(bandwidths,)i fb(B)s(W)965 1599 y fa(t)992 1592 y fj(and)f fb(B)s(W)1158 1599 y fa(r)1177 1592 y fj(.)19 b(If)13 b fb(B)s(W)1332 1599 y fa(t)1359 1592 y fj(=)g fb(B)s(W)1487 1599 y fa(r)1506 1592 y fj(,)f(in)i(principle,)i(no)c (FIF)o(O)0 1649 y(space)17 b(is)g(required,)h(but)e(as)g(mentioned,)h(the)f (transmit)h(and)f(receive)h(bandwidths)g(can)g(b)r(e)f(unequal)i(\(in)f(our)0 1705 y(case,)12 b fb(B)s(W)186 1712 y fa(r)217 1705 y fb(<)h(B)s(W)345 1712 y fa(t)360 1705 y fj(\).)18 b(W)l(e)12 b(can)f(conserv)n(ativ)o(ely)h(b)r(ound) g(the)f(necessary)h(FIF)n(O)g(space)g(b)o(y)g(assuming)g(that)e fb(RT)c(T)0 1762 y fj(=)12 b(0,)g(since)h(cells)h(of)e(the)g(next)g(windo)o(w)g(w)o(ould) g(start)f(arriving)i(as)e(so)q(on)h(as)g(pro)q(cessing)h(on)f(the)g(curren)o (t)g(windo)o(w)0 1818 y(b)r(egins.)20 b(Assuming)15 b fb(R)o(T)6 b(T)20 b fj (=)15 b(0,)f(the)g(maxim)o(um)g(FIF)o(O)g(space)h(is)f fb(QS)1221 1825 y fa (max)1306 1818 y fj(=)f(2)p fb(W)6 b(S)k ff(\000)f fb(B)s(W)1588 1825 y fa (r)1607 1818 y fb(=B)r(W)1709 1825 y fa(t)1724 1818 y fj(.)19 b(Assuming)0 1875 y(conserv)n(ativ)n(ely)f(that)f fb(B)r(W)475 1882 y fa(r)511 1875 y fj (=)h(40)e(Mbps,)h fb(B)s(W)846 1882 y fa(t)878 1875 y fj(=)g(110)f(Mbps,)i (and)f fb(W)6 b(S)20 b fj(=)d(71)g(cells,)h(then)g fb(QS)1741 1882 y fa(max) 1829 1875 y fj(=)g(116)0 1931 y(cells.)23 b(In)17 b(allo)o(wing)g(sim)o(ultan\ eous)f(reception)h(of)e(t)o(w)n(o)h(long)g(messages,)f(the)h(required)h(queue) g(size)g(doubles)g(to)0 1988 y(232)c(cells.)20 b(The)14 b(remaining)h(60)e (cells)i(in)f(the)g(receiv)o(e)g(FIF)n(O)g(are)f(left)h(as)f(a)h(slac)o(k)f (for)g(short)g(messages)g(or)g(initial)0 2044 y(\(single-segment\))j(windo)o (ws.)23 b(The)16 b(slac)o(k)g(will)i(in)f(practice)g(b)r(e)f(higher)h(b)q(eca\ use)g(long)g(messages)e(are)h(not)g(the)0 2100 y(common)f(case,)h(FIF)n(O)g (usage)g(b)o(y)f(eac)o(h)h(long)g(message)f(is)h(exp)r(ected)g(to)f(p)r(eak)g (at)g(di\013erent)g(instan)o(ts,)g fb(RT)6 b(T)21 b fj(is)0 2157 y(not)15 b (zero,)g(and)g(\014nal)h(windo)o(ws)f(are)g(on)g(a)o(v)n(erage)g(only)h(half) g(full.)62 2255 y(Cell)22 b(loss)e(resulting)h(from)e(receiver)h(o)n(v)o(er\ \015o)o(w)g(can)g(still)h(o)q(ccur)g(if)f(a)g(large)g(n)o(um)o(b)q(er)g(of)g (short)f(messages)0 2311 y(or)e(initial)j(windows)d(are)h(sen)o(t)f(to)g(a)h (host)f(that)g(is)i(already)f(receiving)h(long)f(messages.)28 b(The)18 b(rece\ ive)g(FIF)n(O,)0 2368 y(though,)13 b(can)g(bu\013er)g(at)g(least)g(30)g(short) f(messages)h(\(60)f(cells\))j(or)d(60)h(initial)i(windows,)d(and)i(can)f(b)r (e)g(drained)h(in)0 2424 y(less)f(than)f(2.5)f(milliseconds.)22 b(This)13 b (enables)g(more)f(than)g(10000)e(short)i(messages)g(p)q(er)g(second)h(to)e (b)r(e)h(consumed)0 2481 y(in)k(the)f(w)o(orst)f(case)h(of)g(t)o(w)n(o)g(long) g(messages)g(outstanding.)964 2844 y(8)p eop %%Page: 9 12 bop 0 42 a fh(Congestion)18 b(a)o(v)n(oidance)0 169 y fj(Net)o(w)n(ork)e(con\ gestion)g(can)g(o)q(ccur)g(whenev)o(er)g(the)g(aggregate)e(bandwidth)j(destin\ ed)h(for)d(a)g(particular)i(switc)o(h)0 225 y(output)h(p)r(ort)f(exceeds)i (that)f(output)g(p)q(ort's)f(capacit)o(y)l(.)29 b(F)l(or)18 b(example,)h(cong\ estion)g(can)f(o)q(ccur)h(in)g(a)e(single-)0 282 y(switc)o(h)c(net)o(w)n(ork) f(if)i(t)n(w)o(o)e(or)g(more)g(hosts)h(con)n(tinuously)g(\015o)q(o)q(d)g(a)g (single)h(receiv)o(er,)f(ev)o(en)g(though)g(neither)g(sender)0 338 y(exceeds) k(the)e(maxim)o(um)h(bandwidth)h(of)e(its)g(link)j(to)d(the)g(switc)o(h.)22 b (Congestion)15 b(often)g(results)h(in)h(cell)g(lossage)0 395 y(b)r(ecause)e (the)g(switc)o(h)h(has)f(\014nite)h(bu\013ering)g(capacit)o(y)l(.)62 493 y (W)l(e)e(use)f(four)g(simple)i(tec)o(hniques)g(to)d(a)o(v)o(oid)h(congestion:) 20 b(dynamic)14 b(windo)o(w)f(sizing,)i(bu\013er)e(sc)o(heduling)j(at)0 549 y (the)e(receiving)i(host,)e(randomized)h(time-outs)f(with)h(exp)q(onential)f (bac)o(k-o\013,)g(and)g(static)g(tra\016c)g(analysis)h(with)0 606 y(top)q(olo\ gy)i(optimization.)27 b(As)18 b(previously)g(describ)s(ed,)g(the)f(\014rst)g (t)o(w)n(o)g(tec)o(hniques,)i(dynamic)f(window)f(sizing)0 662 y(and)i(bu\013e\ r)g(sc)o(heduling,)i(reduce)f(the)f(p)q(ossibilit)q(y)g(of)f(congestion)h(b)r (et)n(w)o(een)g(hosts)f(connected)i(to)e(the)h(same)0 718 y(switc)o(h.)62 816 y(Bu\013er)14 b(sc)n(heduling)i(at)d(the)g(receiving)i(hosts)e(is)g(p)r(erfor\ med)g(with)h(lo)q(cal)g(information)f(only)m(.)19 b(Consequently)l(,)0 873 y (w)o(e)f(cannot)g(detect)g(tra\016c)f(b)r(et)n(w)o(een)h(intermediate)g(switc) o(hes.)29 b(Congestion)18 b(is)g(p)r(ossible)h(in)g(the)f(switc)o(hes)g(if)0 929 y(man)o(y)e(large)h(messages)f(are)h(transmitted)f(b)r(et)o(w)n(een)h(hos\ ts)g(connected)g(to)g(di\013eren)o(t)g(switc)o(hes.)25 b(In)17 b(this)g(case,) 0 986 y(congestion)c(m)o(ust)g(either)g(b)r(e)g(handled)h(with)g(in)o(termedi\ ate)g(resource)f(reserv)m(ation,)g(link-b)q(y-link)h(\015o)o(w)f(con)o(trol,) 0 1042 y(or)19 b(timeouts)h(and)g(retransmission.)33 b(Because)20 b(the)g(\ \014rst)f(strategy)g(increases)h(connection)h(setup)f(time)g(and)0 1099 y(can) e(w)n(aste)f(net)o(w)o(ork)g(resources)g(during)h(a)g(connection's)g(idle)h (p)q(erio)r(ds,)f(and)f(the)h(second)g(strategy)e(requires)0 1155 y(sophistic\ ated)g(switc)o(h)g(and)f(interface)f(hardw)o(are,)h(w)n(e)h(rely)f(on)g(the)h (third)f(strategy)-5 b(.)62 1253 y(The)16 b(receiv)o(er)g(uses)f(timeout)g (to)g(reclaim)h(its)f(reserv)o(ed)g(FIF)o(O)g(space)h(in)g(cases)f(suc)o(h)g (as)g(sender)h(failure,)f(or)0 1310 y(con)o(tin)o(uous)i(loss)h(of)e(synchron\ ization)h(cells,)i(allowing)e(other)g(long)g(messages)g(to)f(b)r(e)h(received\ .)26 b(The)17 b(sender)0 1366 y(uses)22 b(timeout)f(to)g(detect)h(a)f(lost)h (initial)i(windo)o(w,)f(lost)e(sync)o(hronization)i(cell,)h(or)d(lost)h(segme\ n)o(ts)f(at)g(the)0 1422 y(end)g(of)g(a)f(window)g(\(whic)o(h)h(the)g(receive\ r)g(cannot)f(detect,)i(since)g(it)f(k)o(eeps)g(w)o(aiting)g(for)f(these)h(seg\ men)o(ts\).)0 1479 y(If)h(the)h(sender)f(did)i(not)e(timeout)g(on)g(these)g (ev)o(en)o(ts,)h(remaining)h(windo)o(ws)e(in)h(a\013ected)f(messages)g(w)n (ould)0 1535 y(remain)d(queued)h(for)e(transmission)h(inde\014nitely)o(.)31 b (On)19 b(timeout,)h(the)e(sender)i(retries)f(sev)o(eral)g(times)g(b)q(efore)0 1592 y(ab)q(orting)13 b(the)g(transmission)g(request.)19 b(The)13 b(timeout)f (used)i(b)o(y)e(di\013erent)g(hosts)g(is)i(randomized)f(and)g(increases)0 1648 y(exp)r(onen)o(tially)h(according)f(to)g(the)g(n)o(um)n(b)r(er)g(of)f(retries\ .)20 b(This)13 b(tends)g(to)g(spread)g(retransmissions)g(in)g(time,)h(and)0 1705 y(allo)o(ws)h(forw)n(ard)g(progress)g(in)h(spite)g(of)e(congestion.)62 1803 y(The)f(e\013ectiveness)g(of)f(timeouts)h(is)g(enhanced)i(b)o(y)d(the)h (strict)g(FIF)n(O)h(bu\013ering)f(in)h(the)f(hosts)f(and)h(switc)o(hes.)0 1859 y(A)j(cell)i(cannot)e(linger)h(inde\014nitely)i(in)e(a)f(FIF)o(O,)g(so)g(an)n (y)g(cell)i(not)e(rela)o(y)o(ed)g(within)i(a)d(certain)i(b)r(ounded)g(time)0 1916 y(has)g(b)r(een)h(dropp)r(ed.)27 b(Since)19 b(w)o(e)e(b)r(ound)h(the)g (distance)g(and)g(n)o(um)o(b)q(er)g(of)f(switc)o(hes,)h(w)n(e)g(can)g(also)f (b)r(ound)h(the)0 1972 y(round)13 b(trip)h(time,)g(whic)o(h)g(mak)n(es)f(it)h (p)q(ossible)h(to)d(a)o(v)o(oid)h(spurious)h(timeouts.)20 b(The)13 b(timeout) g(v)n(alue)h(is)f(based)h(on)0 2029 y(the)j(w)o(orst-case)f(assumptions)h(tha\ t)f(the)h(windo)o(w)g(and)g(its)g(ac)o(kno)o(wledgements)f(tra)n(v)o(el)h(thr\ ough)g(5)g(switc)o(hes)0 2085 y(across)e(40)f(km,)h(that)f(queues)j(are)d(nea\ rly)i(full)h(at)e(eac)n(h)h(in)o(termediate)g(switc)o(h,)f(and)h(that)e(end-h\ ost)i(in)o(terrupt)0 2141 y(latency)g(is)g(extremely)f(high)h(\(30)f(msecs\).) 62 2239 y(Lastly)m(,)f(if)g(congestion)g(remains)h(a)e(frequent)g(problem)i (despite)g(the)f(protocol's)g(b)r(est)g(e\013orts,)e(the)i(problem)0 2296 y (remains)g(with)h(the)f(net)o(w)o(ork)f(top)q(ology)h(or)g(link)h(capacities.) 21 b(A)o(t)13 b(this)i(p)q(oint,)e(the)i(only)f(solution)h(is)g(to)e(analyze) 0 2352 y(the)k(tra\016c)f(and)h(recon\014gure)g(the)g(net)n(w)o(ork)f(so)h (that)f(hosts)g(that)g(comm)n(unicate)i(the)f(most)e(with)j(eac)n(h)f(other)0 2409 y(are)e(connected)h(to)e(the)h(same)g(switc)o(h.)20 b(Other)15 b(p)q(oss\ ible)i(rearrangemen)n(ts)e(include)i(using)f(larger)e(switc)o(hes,)h(or)0 2465 y(increasing)h(the)g(n)o(um)n(b)r(er)f(or)f(transmission)i(rate)e(of)h(links) h(b)r(et)n(w)o(een)g(congested)f(switc)o(hes.)20 b(This)c(is)f(analogous)0 2522 y(to)g(the)g(routine)h(managemen)n(t)f(of)g(Ethernet)g(subnet)o(w)o(orks\ ,)f(routers)h(and)g(gatew)n(a)n(ys.)964 2844 y(9)p eop %%Page: 10 13 bop 0 42 a fh(4.4.)52 b(Lo)o(w-o)o(v)n(erhead)17 b(error)f(con)o(trol)i(and) g(reco)n(v)o(ery)0 169 y fj(As)e(with)g(segmen)o(tation)g(and)g(reassem)n(bly) m(,)g(A)l(TM's)f(small)i(cell)h(size)f(imp)r(oses)e(a)h(high)h(demand)f(on)g (an)o(y)g(error)0 225 y(con)o(trol)h(and)f(reco)o(v)o(ery)h(mec)o(hanism.)25 b (A)l(TM)17 b(and)g(AAL)g(3/4)f(require)h(that)g(eac)n(h)g(cell)i(b)q(e)e(c)o (heck)n(ed)h(for)e(errors)0 282 y(in)j(the)g(A)l(TM)f(header,)i(pa)n(yload)f (CR)o(C,)f(segmen)o(t)g(t)o(yp)q(e,)h(and)g(data)f(lengths.)31 b(The)18 b(AAL) h(3/4)f(sp)r(eci\014cation)0 338 y(prescrib)r(es)f(that)g(an)g(en)o(tire)g (message)g(should)h(b)q(e)f(dropp)r(ed)g(in)h(case)f(an)g(error)f(is)i(encoun) o(tered)f(in)h(one)f(of)g(its)0 395 y(received)e(segmen)o(ts.)62 493 y(A)m (TM)h(and)g(AAL)h(3/4)f(error)g(con)o(trol)g(plus)h(the)g(\015o)o(w)f(con)o (trol)g(timeouts)h(and)f(retransmissions)h(describ)s(ed)0 549 y(in)k(the)f (previous)h(section)f(are)g(nearly)g(su\016cient)g(to)f(pro)o(vide)i(a)f(reli\ able)i(transp)q(ort)d(protocol.)35 b(A)20 b(reliable)0 606 y(transp)q(ort)e (can)i(b)q(e)g(pro)n(vided)h(with)e(small)h(additional)h(o)n(v)o(erhead)f fg (if)f fj(it)g(can)h(b)q(e)g(in)o(tegrated)f(with)g(the)h(error)0 662 y(and)i (\015o)n(w)g(con)o(trol)f(mec)o(hanisms.)39 b(In)22 b(the)g(common)f(case)g (of)g(messages)g(b)r(eing)h(deliv)q(ered)g(without)f(error,)0 718 y(the)g(onl\ y)g(additional)h(required)g(service)g(is)f(message)g(sequencing,)i(and)e(end-\ of-message)g(sync)o(hronization)0 775 y(\(ac)n(kno)o(wledgment\))12 b(to)h (indicate)i(that)e(an)g(en)o(tire)h(message)f(has)g(b)r(een)h(correctly)f(rec\ eived.)20 b(W)-5 b(e)14 b(include)i(these)0 831 y(services)g(in)g(our)f(proto\ col)h(implemen)q(tation)f(for)f(the)i(case)f(of)f(sequenced,)j(reliable)g(mes\ sages.)62 929 y(An)o(y)c(error)g(detected)g(b)o(y)g(the)g(receiver)g(causes)g (the)g(receiver)g(to)f(discard)i(the)f(en)o(tire)h(message)e(in)i(which)f(the) 0 986 y(error)k(o)q(ccurred.)29 b(Consequen)o(tly)m(,)18 b(a)g(sender)g(m)o (ust)g(retransmit)f(ev)o(ery)h(message)f(from)g(the)i(b)q(eginning.)30 b(The) 0 1042 y(arriv)m(al)16 b(of)e(a)h(message)f(that)g(is)i(already)f(b)q(eing)h (received)f(is)h(not)e(considered)i(an)f(error,)f(but)h(rather)g(the)g(result) 0 1099 y(of)g(a)g(timeout)g(on)h(the)f(sender)h(\(for)e(example,)i(due)g(to)f (a)g(lost)g(ac)o(kno)n(wledgment\).)20 b(Previously)c(received)g(data)0 1155 y (in)g(the)f(message)g(is)h(simply)g(dropp)r(ed)f(b)o(y)h(the)f(receiver.)62 1253 y(Timeouts)j(are)f(su\016cien)q(t)g(for)g(error)g(con)o(trol)g(and)h(rec\ o)o(v)o(ery)l(,)g(but)g(reco)n(v)o(ery)g(can)g(b)q(e)g(accelerated)h(b)o(y)e (im-)0 1310 y(mediately)j(returning)f(to)f(the)g(sender)i(an)e(indication)j (of)d(the)g(error)g(during)i(ev)o(en)o(ts)e(that)g(are)g(unrelated)i(to)0 1366 y(congestion.)31 b(A)19 b fg(Ne)n(gative)h(A)m(cknow)r(le)m(dgement)f fj(is)h (returned)f(when)g(a)g(message)g(has)f(b)r(een)h(corrupted.)32 b(The)0 1422 y (sender)14 b(retries)f(the)g(message)f(sev)o(eral)h(times)h(but)f(ev)o(en)o (tually)h(giv)o(es)f(up)g(and)g(returns)g(an)g(error)f(to)h(the)g(sending)0 1479 y(pro)q(cess.)62 1577 y(An)h fg(Over\015ow)g(Err)n(or)g fj(indicates)g (a)f(high-lev)q(el)h(resource)f(reserv)n(ation)g(problem)h(that)f(o)q(ccurs)g (whenever)g(the)0 1633 y(receiver)k(do)q(es)g(not)f(ha)o(v)o(e)h(su\016cient) f(bu\013er)h(space)g(to)f(absorb)h(the)f(incoming)j(message.)24 b(The)17 b (protocol)h(do)q(es)0 1690 y(not)g(attempt)f(to)g(reco)o(v)o(er)h(from)f(an)h (o)o(v)n(er\015o)o(w)g(error,)g(as)g(it)g(re\015ects)g(a)g(problem)h(with)f (resource)h(reserv)m(ation)0 1746 y(at)c(or)g(ab)r(o)n(v)o(e)h(the)g(session) g(la)o(y)o(er.)21 b(Instead,)16 b(the)g(protocol)h(returns)e(the)h(error)f (indication)j(in)e(resp)r(onse)g(to)f(the)0 1803 y(initial)i(window)e(request\ .)20 b(The)15 b(sender)h(rela)o(ys)f(the)g(error)g(indication)i(up)f(to)e(the) h(sending)i(pro)q(cess.)62 1901 y(Errors)i(resulting)j(from)d(net)o(w)o(ork)g (congestion)i(where)f(a)g(cell)i(has)e(b)r(een)h(dropp)q(ed)g(b)o(y)f(an)g (intermediate)0 1957 y(switc)o(h)e(are)f(not)h(ac)n(kno)o(wledged)h(b)o(y)e (the)h(receiver.)27 b(These)18 b(include)i(errors)d(in)h(the)g(segmen)o(t)f (t)o(yp)q(e,)h(segmen)o(t)0 2014 y(sequence)c(n)o(um)o(b)r(er,)f(or)g(an)g (inconsistency)i(in)f(the)f(reassem)n(bled)i(message)d(length.)20 b(In)14 b (suc)o(h)g(cases,)f(the)g(o\013end-)0 2070 y(ing)h(message)g(is)g(simply)h (dropp)q(ed.)20 b(The)14 b(sender)g(detects)g(the)g(error)f(through)h(a)f(tim\ eout)h(after)f(a)g(randomized)0 2126 y(delay)f(as)h(describ)r(ed)g(in)h(Secti\ on)g(4.3.)k(The)13 b(sender)g(retries)g(sev)o(eral)g(more)f(times,)h(eac)o (h)g(with)g(an)g(exp)r(onen)o(tially)0 2183 y(increasing)19 b(timeout.)29 b (W)l(e)18 b(do)h(not)e(indicate)j(the)e(error)g(immediately)h(with)g(a)f(nega\ tiv)o(e)g(ac)o(kno)n(wledgement)0 2239 y(from)d(the)i(receiv)o(er)g(to)e(the) h(sender)h(b)r(ecause,)f(in)h(the)f(case)h(of)e(congestion,)i(this)f(w)o(ould) h(tend)f(to)g(sync)o(hronize)0 2296 y(the)f(transmission)h(of)f(di\013eren)o (t)g(senders,)g(preserving)h(the)g(congestion.)0 2457 y fh(4.5.)52 b(Signalin\ g)19 b(and)f(m)n(ultiplexing)0 2585 y fj(The)23 b(hardw)o(are)f(w)o(e)h(used) g(emplo)o(ys)g(the)g(SP)l(ANS)g(signaling)i(protocol.)43 b(This)24 b(protocol) f(allows)f(hosts)h(to)0 2641 y(send)18 b fg(r)n(e)n(quests)f fj(to)h(switc)o (hes)g(to)f(op)q(en)i(or)e(close)i(connections.)29 b(The)18 b(switc)o(hes)g (in)h(turn)e(send)i(corresp)q(onding)0 2698 y fg(indic)m(ations)f fj(to)f(the) i(destination)g(of)e(the)h(connection.)30 b(The)18 b(destination)h(determines) g(whether)g(or)e(not)h(the)952 2844 y(10)p eop %%Page: 11 14 bop 0 42 a fj(indication)18 b(should)e(b)r(e)g(accepted)g(and)g(returns)g (a)f fg(r)n(esp)m(onse)g fj(that)g(is)h(translated)g(in)o(to)f(a)h fg(c)m(on\ \014rmation)g fj(to)f(the)0 98 y(source)k(host.)32 b(Signaling)21 b(messages) e(are)f(sent)g(o)o(v)o(er)h(a)g(w)n(ell-kno)q(wn)g(meta-connection)h(that)e (is)i(main)o(tained)0 154 y(b)o(y)c(hosts)f(and)h(switc)o(h)g(soft)n(w)o(are.) 21 b(Nev)o(ertheless,)c(signaling)g(can)f(incur)h(signi\014cant)f(latency)g (on)g(the)g(order)f(of)0 211 y(tens)g(of)g(milliseconds)j(p)r(er)d(switc)o (h.)20 b(This)c(latency)g(is)g(not)e(a)h(concern)h(when)g(connections)g(remai\ n)g(op)q(en)g(for)e(a)0 267 y(long)i(time)f(\(e.g.,)f(for)h(sequenced)h(pac)o (k)o(ets\),)e(but)i(b)q(ecomes)g(prohibitive)g(if)f(connections)i(are)e(to)f (b)r(e)h(set)g(up)h(on)0 324 y(demand)g(to)e(send)i(sp)r(eci\014c)g(datagrams\ .)62 422 y(T)l(o)j(pro)o(vide)h(e\016cient)f(datagram)e(supp)r(ort,)i(w)n(e)h (transp)q(ort)e(datagrams)f(o)o(v)o(er)i(m)n(ultiplexed)j(connections)0 478 y (\()p fg(lines)p fj(\))11 b(b)q(et)o(w)o(een)i(hosts.)19 b(If)13 b(a)f(datagr\ am)g(is)h(to)g(b)q(e)g(sen)o(t)g(to)f(a)h(host)f(for)h(whic)o(h)g(no)g(line)i (exists,)e(a)g(line)h(is)g(dynam-)0 535 y(ically)k(op)q(ened.)24 b(Subsequen) q(tly)l(,)16 b(an)o(y)g(datagrams)f(sen)o(t)h(b)q(et)o(w)n(een)h(the)g(t)n (w)o(o)f(hosts)f(use)i(the)f(same)g(connection,)0 591 y(so)i(signaling)h(cost\ s)f(are)f(paid)i(only)g(once.)28 b(The)19 b(endp)r(oin)o(ts)f(connected)h(b)o (y)f(the)g(line)h(are)f(also)g(dynamically)0 648 y(created.)h(Since)d(the)e (line)i(initiator)e(do)r(es)g(not)f(kno)o(w)h(the)g(endp)r(oint)f(that)g(will) j(b)r(e)e(assigned)g(to)g(the)g(line)i(at)d(the)0 704 y(p)r(eer)18 b(host,)g (it)g(names)g(instead)h(a)f fg(virtual)i(endp)m(oint)e fj(as)g(the)g(destinat\ ion)h(endp)r(oint.)28 b(The)19 b(signaling)g(imple-)0 760 y(men)o(tation)14 b (treats)f(this)h(case)g(sp)r(ecially)n(,)g(dynamically)i(creating)e(the)g(lin\ e's)h(endp)r(oint)e(at)h(the)g(p)q(eer)h(host.)k(The)0 817 y(initiator)13 b (learns)g(the)f(identit)o(y)g(of)g(the)h(line)h(endp)r(oin)o(t)e(at)g(the)g (p)r(eer)g(at)g(the)g(time)h(it)f(receives)h(the)f(corresp)q(onding)0 873 y (line)j(op)q(en)f(indication)h(in)f(the)f(rev)o(erse)g(direction.)21 b(The)13 b(proto)q(col)g(soft)n(w)o(are)g(reclaims)h(lines)h(when)e(the)h(n)o(um)n(b)r (er)0 930 y(of)h(free)g(endp)r(oints)f(on)h(a)g(host)g(drops)g(b)r(elo)o(w)g (a)g(threshold.)62 1028 y(The)j(bu\013er)g(for)f(a)h(receiv)o(ed)h(message)e (is)h(allo)r(cated)g(directly)h(from)e(the)h(bu\013er)g(area)f(of)g(the)h(des\ tination)0 1084 y(endp)r(oint,)f(rather)g(than)g(the)h(line)h(endp)r(oin)o (t.)27 b(Multiple)19 b(messages)e(ma)o(y)g(b)r(e)g(received)h(sim)o(ultaneous\ ly)h(o)n(v)o(er)0 1141 y(di\013erent)10 b(line)i(endp)r(oints)e(for)g(the)g (same)h(destination)g(endp)r(oint.)18 b(Eac)n(h)11 b(line)h(endp)s(oin)o(t)e (reassem)o(bles)h(a)f(message)0 1197 y(on)15 b(a)g(di\013erent)f(bu\013er,)h (although)h(all)g(bu\013ers)f(are)g(allo)r(cated)g(from)f(the)i(same)f(bu\013\ er)g(area.)0 1359 y fh(4.6.)52 b(In)n(tegration)19 b(with)f(In)n(ternet)g(pro\ to)q(cols)0 1486 y fj(In)o(ternet)h(proto)q(cols)h(ma)n(y)f(b)r(e)g(la)o(y)o (ered)h(on)f(top)g(of)g(our)g(net)o(w)n(ork)g(interface)g(to)f(allow)h(comm)n (unication)i(o)n(v)o(er)0 1543 y(heterogeneous)e(or)g(wide-area)h(net)o(w)n (orks.)32 b(IP)19 b(comm)o(unicates)h(through)f(a)g(sp)q(ecial,)j(w)n(ell-kno\ wn)e(endp)r(oin)o(t)0 1599 y(pre-allo)r(cated)i(on)f(eac)o(h)h(no)q(de.)39 b (The)22 b(IP)g(endp)r(oin)o(t)f(uses)h(the)f(ra)o(w)g(datagram)f(protocol;)25 b(\015o)o(w)c(and)h(error)0 1655 y(con)o(trol)17 b(m)o(ust)h(b)q(e)g(pro)n (vided)h(b)o(y)f(higher)h(lev)o(el)g(soft)n(w)n(are)e(\(e.g.)27 b(TCP\))17 b (rather)g(than)h(the)f(net)o(w)o(ork)g(interface.)0 1712 y(The)e(TCP/IP)g(sta\ c)n(k)g(can)g(b)q(e)g(implemen)q(ted)g(inside)i(or)d(outside)i(the)f(k)n(erne\ l;)h(in)g(the)f(latter)f(case)h(a)g(privileged)0 1768 y(call)h(is)g(o\013ered) f(for)f(the)i(IP)f(task)g(to)f(register)h(itself)i(as)d(the)i(o)n(wner)f(of)g (the)g(IP)h(endp)r(oin)o(t.)62 1866 y(W)l(e)c(pro)n(vide)h(t)n(w)o(o)e(featur\ es)g(to)g(impro)o(v)o(e)h(the)g(p)q(erformance)f(of)g(integration)g(of)g(Inte\ rnet)g(protocols)h(with)g(our)0 1923 y(net)o(w)n(ork)g(in)o(terface:)18 b(fas\ t)11 b(address)h(lo)r(okup,)g(and)h(fast)e(dem)o(ultiplexing.)21 b(An)12 b (address)g(lo)r(okup)g(service)h(maps)f(IP)0 1979 y(addresses)f(or)f(host)g (names)h(into)f(ph)o(ysical)i(A)l(TM)e(addresses.)19 b(The)11 b(session)g(la) o(y)o(er)g(soft)n(w)n(are)g(can)g(use)g(this)g(service)0 2036 y(to)16 b(b)o (ypass)h(TCP/IP)g(la)o(y)o(ers)g(when)h(the)f(A)l(TM)g(address)g(lo)q(okup)h (succeeds.)27 b(F)-5 b(ast)17 b(pac)o(k)n(et)g(demultiplexing)i(is)0 2092 y (made)d(p)r(ossible)i(b)o(y)e fg(p)n(ort)i(r)n(e)m(gistr)n(ation)p fj(.)23 b (Bu\013ers)17 b(for)f(incoming)h(IP)g(messages)f(are)g(normally)h(allo)r(cate\ d)g(from)0 2149 y(the)d(IP)h(endp)r(oint)f(bu\013er)g(area,)g(and)g(deliv)q (ered)g(to)g(threads)g(w)o(aiting)h(on)f(the)h(IP)f(endp)r(oint.)19 b(Ho)o (w)n(ev)o(er,)14 b(the)h(IP)0 2205 y(task)d(can)i(register)e(sp)r(eci\014c)i (A)m(TM)e(endp)r(oints)g(for)h(giv)o(en)g(TCP)g(or)f(UDP)h(p)r(orts.)18 b(If) 13 b(an)g(IP)h(message)e(is)i(received)0 2261 y(and)j(there)g(is)g(a)f(sp)r (eci\014c)i(endp)r(oin)o(t)f(registered)g(for)f(the)h(TCP)f(or)g(UDP)h(p)q (ort)f(indicated)i(in)g(the)f(header,)g(the)0 2318 y(bu\013er)i(for)g(the)g (message)g(is)h(allo)r(cated)f(from)f(the)i(bu\013er)f(area)g(of)f(the)i(sp)q (eci\014c)h(endp)r(oin)o(t,)f(and)f(not)g(of)g(the)0 2374 y(IP)e(endp)r(oin)o (t.)24 b(If)17 b(the)f(IP)h(task)f(arranges)g(for)g(the)g(sp)r(eci\014c)i(end\ p)r(oin)o(t)f(to)e(b)r(e)i(co-o)n(wned)g(b)o(y)g(the)f(applications)0 2431 y (ha)o(ving)g(access)f(to)g(the)g(giv)o(en)h(TCP)f(or)g(UDP)g(p)r(ort,)f(a)h (cop)o(y)g(is)h(a)n(v)o(oided,)g(since)h(the)e(bu\013er)g(is)h(allo)r(cated)g (from)0 2487 y(an)e(area)f(already)h(mapp)q(ed)h(to)e(the)h(correct)f(applica\ tion)i(address)f(spaces.)20 b(F)l(urthermore,)13 b(suc)o(h)h(messages)f(are)0 2544 y(delivered)j(to)f(an)o(y)g(threads)h(that)e(are)i(w)n(aiting)g(on)g(the) f(sp)r(eci\014c)i(endp)r(oin)o(t.)k(These)16 b(threads)f(ma)n(y)h(b)q(e)g(run\ ning)0 2600 y(on)f(the)g(end)h(application)h(address)e(space,)g(further)g(red\ ucing)i(the)e(dem)o(ultiplexing)j(latency)e([20)o(].)952 2844 y(11)p eop %%Page: 12 15 bop 0 0 a 28417720 18945146 3289088 3289088 26970521 19866091 startTexFig %%BeginDocument: lat.ps /gnudict 40 dict def gnudict begin /Color false def /gnulinewidth 5.000 def /vshift -23 def /dl {10 mul} def /hpt 31.5 def /vpt 31.5 def /vpt2 vpt 2 mul def /hpt2 hpt 2 mul def /Lshow { currentpoint stroke moveto 0 vshift rmoveto show } def /Rshow { currentpoint stroke moveto dup stringwidth pop neg vshift rmoveto show } def /Cshow { currentpoint stroke moveto dup stringwidth pop -2 div vshift rmoveto show } def /DL { Color {setrgbcolor [] 0 setdash pop} {pop pop pop 0 setdash} ifelse } def /BL { stroke gnulinewidth 2 mul setlinewidth } def /AL { stroke gnulinewidth 2 div setlinewidth } def /PL { stroke gnulinewidth setlinewidth } def /LTb { BL [] 0 0 0 DL } def /LTa { AL [1 dl 2 dl] 0 setdash 0 0 0 setrgbcolor } def /LT0 { PL [] 0 1 0 DL } def /LT1 { PL [4 dl 2 dl] 0 0 1 DL } def /LT2 { PL [2 dl 3 dl] 1 0 0 DL } def /LT3 { PL [1 dl 1.5 dl] 1 0 1 DL } def /LT4 { PL [5 dl 2 dl 1 dl 2 dl] 0 1 1 DL } def /LT5 { PL [4 dl 3 dl 1 dl 3 dl] 1 1 0 DL } def /LT6 { PL [2 dl 2 dl 2 dl 4 dl] 0 0 0 DL } def /LT7 { PL [2 dl 2 dl 2 dl 2 dl 2 dl 4 dl] 1 0.3 0 DL } def /LT8 { PL [2 dl 2 dl 2 dl 2 dl 2 dl 2 dl 2 dl 4 dl] 0.5 0.5 0.5 DL } def /M {moveto} def /L {lineto} def /P { stroke [] 0 setdash currentlinewidth 2 div sub moveto 0 currentlinewidth rlineto stroke } def /D { stroke [] 0 setdash 2 copy vpt add moveto hpt neg vpt neg rlineto hpt vpt neg rlineto hpt vpt rlineto hpt neg vpt rlineto closepath stroke P } def /A { stroke [] 0 setdash vpt sub moveto 0 vpt2 rlineto currentpoint stroke moveto hpt neg vpt neg rmoveto hpt2 0 rlineto stroke } def /B { stroke [] 0 setdash 2 copy exch hpt sub exch vpt add moveto 0 vpt2 neg rlineto hpt2 0 rlineto 0 vpt2 rlineto hpt2 neg 0 rlineto closepath stroke P } def /C { stroke [] 0 setdash exch hpt sub exch vpt add moveto hpt2 vpt2 neg rlineto currentpoint stroke moveto hpt2 neg 0 rmoveto hpt2 vpt2 rlineto stroke } def /T { stroke [] 0 setdash 2 copy vpt 1.12 mul add moveto hpt neg vpt -1.62 mul rlineto hpt 2 mul 0 rlineto hpt neg vpt 1.62 mul rlineto closepath stroke P } def /S { 2 copy A C} def end gnudict begin gsave 50 50 translate 0.100 0.100 scale 0 setgray /Courier findfont 70 scalefont setfont newpath LTa 504 246 M 3453 246 L 504 246 M 504 2344 L LTb 504 246 M 567 246 L 3453 246 M 3390 246 L 462 246 M (0) Rshow 504 546 M 567 546 L 3453 546 M 3390 546 L 462 546 M (2000) Rshow 504 845 M 567 845 L 3453 845 M 3390 845 L 462 845 M (4000) Rshow 504 1145 M 567 1145 L 3453 1145 M 3390 1145 L 462 1145 M (6000) Rshow 504 1445 M 567 1445 L 3453 1445 M 3390 1445 L 462 1445 M (8000) Rshow 504 1745 M 567 1745 L 3453 1745 M 3390 1745 L 462 1745 M (10000) Rshow 504 2044 M 567 2044 L 3453 2044 M 3390 2044 L 462 2044 M (12000) Rshow 504 2344 M 567 2344 L 3453 2344 M 3390 2344 L 462 2344 M (14000) Rshow 504 246 M 504 309 L 504 2344 M 504 2281 L 504 176 M (0) Cshow 873 246 M 873 309 L 873 2344 M 873 2281 L 873 176 M (1000) Cshow 1241 246 M 1241 309 L 1241 2344 M 1241 2281 L 1241 176 M (2000) Cshow 1610 246 M 1610 309 L 1610 2344 M 1610 2281 L 1610 176 M (3000) Cshow 1979 246 M 1979 309 L 1979 2344 M 1979 2281 L 1979 176 M (4000) Cshow 2347 246 M 2347 309 L 2347 2344 M 2347 2281 L 2347 176 M (5000) Cshow 2716 246 M 2716 309 L 2716 2344 M 2716 2281 L 2716 176 M (6000) Cshow 3084 246 M 3084 309 L 3084 2344 M 3084 2281 L 3084 176 M (7000) Cshow 3453 246 M 3453 309 L 3453 2344 M 3453 2281 L 3453 176 M (8000) Cshow LTb 504 246 M 3453 246 L 3453 2344 L 504 2344 L 504 246 L 1978 2414 M (Round Trip Latency) Cshow 2531 21 M (Message size \(bytes\)) Lshow 172 2194 M (usec) Lshow LT0 LT0 1610 2044 M (Ultrix TCP/IP) Rshow 1652 2044 M 1778 2044 L 505 435 M 505 435 L 511 437 L 533 463 L 578 499 L 688 589 L 1020 761 L 1979 1284 L 3453 2148 L 1694 2044 D 505 435 D 511 437 D 533 463 D 578 499 D 688 589 D 1020 761 D 1979 1284 D 3453 2148 D LT1 1610 1974 M (4.4 BSD Unix TCP/IP) Rshow 1652 1974 M 1778 1974 L 505 399 M 505 399 L 511 402 L 533 439 L 578 474 L 688 567 L 1020 692 L 1979 1129 L 3453 1840 L 1694 1974 A 505 399 A 511 402 A 533 439 A 578 474 A 688 567 A 1020 692 A 1979 1129 A 3453 1840 A LT2 1610 1904 M (Sequenced Packet Protocol) Rshow 1652 1904 M 1778 1904 L 517 269 M 517 269 L 533 271 L 550 283 L 566 285 L 598 290 L 693 303 L 881 330 L 1259 380 L 2014 480 L 3453 678 L 1694 1904 B 517 269 B 533 271 B 550 283 B 566 285 B 598 290 B 693 303 B 881 330 B 1259 380 B 2014 480 B stroke grestore end showpage %%EndDocument endTexFig 1298 x fj(Figure)12 b(1:)18 b(Pro)q(cess-to-pro)q(cess)12 b(round\ -trip)g(latencies)i(using)e(TCP/IP)g(and)g(using)g(our)g(in)o(tegrated)g(sequ\ enced)0 1354 y(pac)o(k)o(et)j(protocol.)0 1488 y fh(5.)52 b(P)n(erformance)0 1631 y fj(In)15 b(this)g(section)g(w)o(e)g(presen)o(t)f(the)h(p)q(erformance) f(c)o(haracteristics)h(of)f(our)h(protocol)g(implemen)q(tation)f(in)i(terms)0 1687 y(of)e(throughput)h(and)f(latency)m(.)20 b(All)c(measuremen)o(ts)e(are)g (b)r(et)n(w)o(een)h(t)o(w)n(o)f(DECstation)g(5000/200)e(w)o(orkstations)0 1744 y(connected)k(directly)h(b)o(y)e(F)l(ore)g(Systems)g(TCA)f(100)h(A)l(TM)g(in) o(terfaces.)1237 1727 y fd(4)62 1842 y fj(Our)21 b(proto)q(col)g(suite)g(is)g (implemen)q(ted)f(with)h(ab)q(out)f(4100)g(lines)i(of)e(C)g(co)q(de)h(\(not)f (including)k(signaling)0 1898 y(supp)r(ort\).)18 b(Determining)13 b(which)g (part)f(of)g(our)g(implemen)q(tation)g(corresp)q(onds)h(to)f(which)g(conceptu\ al)i(protocol)0 1955 y(la)o(y)o(er)i(is)g(di\016cult)i(b)q(ecause)f(the)f(imp\ lemen)q(tation)f(is)i(integrated.)k(Roughly)m(,)16 b(3300)f(lines)j(are)e(res\ p)q(onsible)i(for)0 2011 y(device)f(managemen)n(t,)d(and)i(the)f(A)l(TM,)f (SAR)i(and)f(CS)h(\\la)n(y)o(ers.")k(Our)15 b(higher-lev)q(el)g(extensions)h (to)f(the)g(base)0 2067 y(A)l(TM)g(services)h(constitute)g(slightly)f(less)h (than)f(15%)g(of)g(the)g(co)q(de.)21 b(F)l(ew)n(er)16 b(than)f(3%)g(of)f(the) i(co)q(de)g(deals)g(with)0 2124 y(reliable)f(messaging.)k(This)14 b(con\014rm\ s)f(that)f(higher-level)i(functionalit)q(y)e(can)i(b)q(e)f(integrated)f(into) g(the)h(net)o(w)n(ork)0 2180 y(interface)h(level)i(without)f(adding)h(undue)h (complexit)o(y)m(.)62 2278 y(Pro)q(cess-to-pro)q(cess)h(\(user-lev)o(el\))h (round-trip)f(times)g(comparing)g(our)f(sequenced)i(pac)o(k)o(et)e(proto)q (col)h(and)0 2335 y(t)n(w)o(o)i(TCP/IP)h(implementations,)h(one)f(from)f(Ultr\ ix)h(and)g(one)g(from)f(4.4)g(BSD,)g(are)h(sho)n(wn)g(in)h(Figure)f(1.)0 2391 y(\(The)f(TCP)f(measuremen)o(ts)g(are)h(tak)n(en)g(from)f([33)o(].\))33 b(The) 19 b(round-trip)i(time)f(for)f(4-b)o(yte)g(messages)h(is)g(152)0 2448 y(micro\ seconds)j(for)e(our)h(sequenced)i(pac)o(k)n(et)e(proto)q(col,)i(1261)d(micros\ econds)h(using)h(Ultrix)g(TCP/IP)l(,)f(and)0 2504 y(1021)15 b(microseconds)j (using)f(BSD)f(4.4)g(TCP/IP)l(.)g(The)h(sequenced)h(pac)o(k)n(et)f(protocol)g (requires)g(t)o(w)n(o)f(messages)0 2561 y(\(sen)o(t)22 b(b)o(y)g(the)g(user\)) g(and)h(t)n(w)o(o)f(ac)n(kno)o(wledgments)g(\(sen)o(t)f(b)o(y)i(the)f(driv)o (er,)i(but)f(mostly)f(o)n(v)o(erlapp)r(ed)h(with)0 2611 y 780 2 v 52 2668 a fc (4)69 2684 y fi(W)m(e)17 b(do)h(not)f(include)i(switch)e(delay)g(in)h(our)f (throughput)i(and)f(latencies)h(to)d(facilitate)j(comparisons)g(to)e(other)h (published)0 2730 y(n)o(umb)q(ers)13 b(on)g(the)g(same)h(hardw)o(are.)952 2844 y fj(12)p eop %%Page: 13 16 bop 0 42 a fj(user)18 b(pro)q(cessing\).)27 b(In)18 b(addition)h(to)d(latenc\ y)j(m)n(uc)o(h)f(lo)o(w)o(er)f(than)h(that)e(of)h(TCP/IP)l(,)g(our)h(in)o(ter\ face)f(ac)o(hieves)0 98 y(considerably)e(higher)g(throughput,)e(as)g(indicate\ d)j(b)o(y)e(the)f(di\013erent)g(inclinations)j(of)e(the)f(curv)o(es)h(in)h (Figure)f(1.)0 154 y(W)l(e)j(b)q(eliev)q(e)g(our)f(b)r(etter)g(p)q(erformance) h(can)g(b)q(e)g(attributed)g(primarily)h(to)e(the)h(integrated)f(la)o(y)n(er) h(pro)q(cessing)0 211 y(organization)f(of)g(our)h(implementation,)f(and)h(to) f(our)g(in)o(tegrated)g(bu\013er)g(managemen)o(t.)23 b(The)17 b(large)f(laten\ cy)0 267 y(and)i(bandwidth)i(di\013erence)f(b)r(et)n(w)o(een)g(our)f(protocol) h(and)f(b)r(oth)f(v)o(ersions)i(of)f(TCP/IP)f(argues)h(strongly)g(in)0 324 y (fa)n(v)o(or)g(of)f(b)o(ypassing)i(higher-lev)q(el)f(proto)q(cols)g(whenever) g(p)r(ossible.)30 b(The)18 b(p)q(erformance)g(disparity)g(supp)r(orts)0 380 y (our)e(decision)i(to)e(build)i(protocols)f(and)f(op)r(erating)g(system)g(in)o (terfaces)g(from)g(scratc)n(h,)g(and)h(con\014rms)f(earlier)0 437 y(predictio\ ns)h(that)d(go)q(o)q(d)h(proto)q(col)g(p)r(erformance)g(can)g(b)r(e)g(obtaine\ d)h(with)f(relatively)h(simple)g(hardw)o(are)f([4)o(].)62 535 y(Figure)e(2)f (sho)n(ws)g(the)h(throughput)f(obtained)h(with)f(di\013erent)g(message)g(size\ s)h(for)f(our)g(three)g(A)l(TM)g(protocol)0 591 y(options.)34 b(Throughput)20 b(for)f(sequenced)i(pac)o(k)o(ets)f(and)g(datagrams)e(con)o(v)o(erge)h(rapidl\ y)i(to)f(their)g(maxim)o(um)0 648 y(sustained)c(throughput)e(of)h(48)f(Mbps.) 20 b(Both)15 b(curv)o(es)g(ha)n(v)o(e)g(their)h(knee)f(at)f(ab)r(out)g(4)h (KB.)g(F)-5 b(or)15 b(messages)f(at)g(or)0 704 y(ab)q(o)o(v)n(e)k(4)e(KB,)h (the)g(throughput)g(o)o(v)n(erhead)h(of)e(\015o)o(w)h(con)o(trol,)f(error)h (and)g(sequence)h(con)o(trol)f(and)g(connection)0 760 y(m)o(ultiplexing)24 b (amoun)o(ts)d(to)g(less)i(than)e(5%.)39 b(The)22 b(maxim)o(um)g(throughput)f (that)g(could)i(theoretically)g(b)r(e)0 817 y(ac)o(hiev)o(ed)17 b(at)e(the)h (receive)f(side,)i(considering)g(only)g(the)f(I/O)g(bus)g(bandwidth)h(and)f (loss)g(due)g(to)g(headers)g(and)0 873 y(trailers)g(while)h(neglecting)g(p)q (ossibly)g(signi\014cant)e(cac)o(he)h(e\013ects,)e(is)i(62)f(Mbps.)21 b(Our) 16 b(end-to-end)g(throughput)0 930 y(ac)o(hiev)o(es)f(76%)e(of)h(this)h(upp)r (er)f(b)r(ound.)20 b(The)14 b(throughput)g(curv)o(e)h(for)e(the)i(ra)n(w)f (proto)q(col)g(stops)g(at)g(a)g(relativ)o(ely)0 986 y(small)e(message)f(size) i(b)q(ecause)f(it)g(is)g(not)f(\015o)n(w-con)o(trolled,)i(causing)f(receiver) f(FIF)o(O)g(o)o(v)o(errun)g(at)g(larger)g(message)0 1043 y(sizes.)0 1093 y 28417720 18945146 3289088 3289088 26970521 19866091 startTexFig %%BeginDocument: through.ps /gnudict 40 dict def gnudict begin /Color false def /gnulinewidth 5.000 def /vshift -23 def /dl {10 mul} def /hpt 31.5 def /vpt 31.5 def /vpt2 vpt 2 mul def /hpt2 hpt 2 mul def /Lshow { currentpoint stroke moveto 0 vshift rmoveto show } def /Rshow { currentpoint stroke moveto dup stringwidth pop neg vshift rmoveto show } def /Cshow { currentpoint stroke moveto dup stringwidth pop -2 div vshift rmoveto show } def /DL { Color {setrgbcolor [] 0 setdash pop} {pop pop pop 0 setdash} ifelse } def /BL { stroke gnulinewidth 2 mul setlinewidth } def /AL { stroke gnulinewidth 2 div setlinewidth } def /PL { stroke gnulinewidth setlinewidth } def /LTb { BL [] 0 0 0 DL } def /LTa { AL [1 dl 2 dl] 0 setdash 0 0 0 setrgbcolor } def /LT0 { PL [] 0 1 0 DL } def /LT1 { PL [4 dl 2 dl] 0 0 1 DL } def /LT2 { PL [2 dl 3 dl] 1 0 0 DL } def /LT3 { PL [1 dl 1.5 dl] 1 0 1 DL } def /LT4 { PL [5 dl 2 dl 1 dl 2 dl] 0 1 1 DL } def /LT5 { PL [4 dl 3 dl 1 dl 3 dl] 1 1 0 DL } def /LT6 { PL [2 dl 2 dl 2 dl 4 dl] 0 0 0 DL } def /LT7 { PL [2 dl 2 dl 2 dl 2 dl 2 dl 4 dl] 1 0.3 0 DL } def /LT8 { PL [2 dl 2 dl 2 dl 2 dl 2 dl 2 dl 2 dl 4 dl] 0.5 0.5 0.5 DL } def /M {moveto} def /L {lineto} def /P { stroke [] 0 setdash currentlinewidth 2 div sub moveto 0 currentlinewidth rlineto stroke } def /D { stroke [] 0 setdash 2 copy vpt add moveto hpt neg vpt neg rlineto hpt vpt neg rlineto hpt vpt rlineto hpt neg vpt rlineto closepath stroke P } def /A { stroke [] 0 setdash vpt sub moveto 0 vpt2 rlineto currentpoint stroke moveto hpt neg vpt neg rmoveto hpt2 0 rlineto stroke } def /B { stroke [] 0 setdash 2 copy exch hpt sub exch vpt add moveto 0 vpt2 neg rlineto hpt2 0 rlineto 0 vpt2 rlineto hpt2 neg 0 rlineto closepath stroke P } def /C { stroke [] 0 setdash exch hpt sub exch vpt add moveto hpt2 vpt2 neg rlineto currentpoint stroke moveto hpt2 neg 0 rmoveto hpt2 vpt2 rlineto stroke } def /T { stroke [] 0 setdash 2 copy vpt 1.12 mul add moveto hpt neg vpt -1.62 mul rlineto hpt 2 mul 0 rlineto hpt neg vpt 1.62 mul rlineto closepath stroke P } def /S { 2 copy A C} def end gnudict begin gsave 50 50 translate 0.100 0.100 scale 0 setgray /Courier findfont 70 scalefont setfont newpath LTa 504 246 M 3453 246 L 504 246 M 504 2344 L LTb 504 246 M 567 246 L 3453 246 M 3390 246 L 462 246 M (0) Rshow 504 456 M 567 456 L 3453 456 M 3390 456 L 462 456 M (5) Rshow 504 666 M 567 666 L 3453 666 M 3390 666 L 462 666 M (10) Rshow 504 875 M 567 875 L 3453 875 M 3390 875 L 462 875 M (15) Rshow 504 1085 M 567 1085 L 3453 1085 M 3390 1085 L 462 1085 M (20) Rshow 504 1295 M 567 1295 L 3453 1295 M 3390 1295 L 462 1295 M (25) Rshow 504 1505 M 567 1505 L 3453 1505 M 3390 1505 L 462 1505 M (30) Rshow 504 1715 M 567 1715 L 3453 1715 M 3390 1715 L 462 1715 M (35) Rshow 504 1924 M 567 1924 L 3453 1924 M 3390 1924 L 462 1924 M (40) Rshow 504 2134 M 567 2134 L 3453 2134 M 3390 2134 L 462 2134 M (45) Rshow 504 2344 M 567 2344 L 3453 2344 M 3390 2344 L 462 2344 M (50) Rshow 504 246 M 504 309 L 504 2344 M 504 2281 L 504 176 M (0) Cshow 951 246 M 951 309 L 951 2344 M 951 2281 L 951 176 M (10000) Cshow 1398 246 M 1398 309 L 1398 2344 M 1398 2281 L 1398 176 M (20000) Cshow 1844 246 M 1844 309 L 1844 2344 M 1844 2281 L 1844 176 M (30000) Cshow 2291 246 M 2291 309 L 2291 2344 M 2291 2281 L 2291 176 M (40000) Cshow 2738 246 M 2738 309 L 2738 2344 M 2738 2281 L 2738 176 M (50000) Cshow 3185 246 M 3185 309 L 3185 2344 M 3185 2281 L 3185 176 M (60000) Cshow LTb 504 246 M 3453 246 L 3453 2344 L 504 2344 L 504 246 L 1978 2414 M (Throughput) Cshow 2515 -5 M (Message size \(bytes\)) Lshow 227 2218 M (Mbps) Lshow LT0 LT0 3140 666 M (Raw ATM + AAL3/4 + CS Protocol) Rshow 3182 666 M 3308 666 L 506 414 M 506 414 L 508 582 L 510 708 L 512 791 L 515 1001 L 527 1337 L 550 1673 L 596 1924 L 687 2050 L 870 2176 L 3224 666 D 506 414 D 508 582 D 510 708 D 512 791 D 515 1001 D 527 1337 D 550 1673 D 596 1924 D 687 2050 D 870 2176 D LT1 3140 596 M (Sequenced Packet Protocol) Rshow 3182 596 M 3308 596 L 506 414 M 506 414 L 508 582 L 510 582 L 512 666 L 515 833 L 527 1127 L 550 1463 L 596 1799 L 687 2008 L 870 2092 L 1236 2218 L 1968 2218 L 3432 2260 L 3224 596 A 506 414 A 508 582 A 510 582 A 512 666 A 515 833 A 527 1127 A 550 1463 A 596 1799 A 687 2008 A 870 2092 A 1236 2218 A 1968 2218 A 3432 2260 A LT2 3140 526 M (Datagram Protocol) Rshow 3182 526 M 3308 526 L 505 372 M 505 372 L 507 540 L 509 582 L 511 666 L 515 833 L 527 1127 L 550 1463 L 596 1757 L 687 1966 L 870 2092 L 1236 2176 L 1968 2218 L 3432 2260 L 3224 526 B 505 372 B 507 540 B 509 582 B 511 666 B 515 833 B 527 1127 B 550 1463 B 596 1757 B 687 1966 B 870 2092 B 1236 2176 B 1968 2218 B 3432 2260 B stroke grestore end showpage %%EndDocument endTexFig 262 2391 a(Figure)k(2:)20 b(Process-to-pro)q(cess)c(throughput)f (using)h(our)f(in)o(tegrated)g(protocols.)62 2506 y(T)l(o)20 b(determine)i (the)e(relative)g(o)o(v)o(erheads)g(of)g(\015o)o(w)g(con)o(trol,)h(error)f (and)h(sequence)g(con)o(trol)f(and)h(connec-)0 2562 y(tion)14 b(m)o(ultiplexi\ ng,)i(w)o(e)e(measured)g(the)g(round-trip)h(times)f(and)g(throughput)f(when)i (sending)g(messages)e(using)0 2619 y(sequenced)18 b(pac)o(k)o(ets)e(\(reliabl\ e,)j(\015o)o(w-con)n(trolled\),)f(datagrams)d(\(\015o)n(w-con)o(trolled\),)i (and)g(a)g(ra)n(w)g(protocol)g(im-)0 2675 y(plementing)k(only)g(the)f(A)l(TM,) g(SAR)h(and)f(CS)h(la)o(y)o(ers)f(o)n(v)o(er)g(exclusiv)q(e)g(connections.)37 b(Figure)20 b(3)g(sho)o(ws)g(the)0 2731 y(round-trip)d(times)f(for)g(messages) f(smaller)i(than)f(800)f(b)o(ytes.)23 b(Up)16 b(to)g(80)f(b)o(ytes,)h(the)g (round-trip)h(time)f(for)g(se-)952 2844 y(13)p eop %%Page: 14 17 bop 0 42 a fj(quenced)14 b(pac)o(k)n(ets)f(is)g(only)g(marginally)g(higher)g (than)g(that)e(for)h(the)h(ra)n(w)f(proto)q(col.)19 b(This)13 b(is)g(b)r(ecau\ se)f(reliabili)q(t)q(y)0 98 y(is)k(integrated)f(at)h(the)g(net)n(w)o(ork)f (interface)h(lev)o(el,)h(and)f(the)g(only)h(non-o)n(v)o(erlapp)r(ed)g(cost)e (of)h(the)g(ac)n(kno)o(wledge-)0 154 y(men)o(t)j(is)h(that)f(of)g(sending)h (one)g(sync)o(hronization)g(cell.)35 b(The)19 b(round-trip)h(time)g(for)f(dat\ agrams)f(is)i(slightly)0 211 y(higher)15 b(b)q(ecause)g(of)f(m)n(ultiplexing/\ dem)r(ultiplexing)j(o)n(v)o(erheads.)j(Messages)13 b(larger)h(than)g(80)f(b)o (ytes)h(sen)o(t)g(b)q(oth)0 267 y(using)e(sequenced)h(pac)o(k)o(ets)e(and)h (datagrams)e(are)h(\015o)o(w-con)o(trolled,)i(causing)f(the)g(step)f(seen)h (in)h(the)f(\014gure.)18 b(This)0 324 y(step)f(is)g(due)h(to)e(the)h(non-o)o (v)n(erlapp)r(ed)h(sync)o(hronization)f(at)g(the)g(initial)h(window.)24 b(Oth\ er)17 b(synchronizations,)0 380 y(ho)o(w)n(ev)o(er,)e(are)f(o)n(v)o(erlapp)r (ed,)h(and)g(the)g(step)f(remains)h(constan)o(t)f(for)g(higher)i(message)e (sizes,)h(allowing)g(the)f(\015o)o(w)0 437 y(con)o(trol)h(o)n(v)o(erhead)h (to)e(b)r(e)h(amortized)g(with)h(larger)f(transmission)g(units.)0 495 y 28417720 18945146 3289088 3289088 26970521 19866091 startTexFig %%BeginDocument: inc_lat.ps /gnudict 40 dict def gnudict begin /Color false def /gnulinewidth 5.000 def /vshift -23 def /dl {10 mul} def /hpt 31.5 def /vpt 31.5 def /vpt2 vpt 2 mul def /hpt2 hpt 2 mul def /Lshow { currentpoint stroke moveto 0 vshift rmoveto show } def /Rshow { currentpoint stroke moveto dup stringwidth pop neg vshift rmoveto show } def /Cshow { currentpoint stroke moveto dup stringwidth pop -2 div vshift rmoveto show } def /DL { Color {setrgbcolor [] 0 setdash pop} {pop pop pop 0 setdash} ifelse } def /BL { stroke gnulinewidth 2 mul setlinewidth } def /AL { stroke gnulinewidth 2 div setlinewidth } def /PL { stroke gnulinewidth setlinewidth } def /LTb { BL [] 0 0 0 DL } def /LTa { AL [1 dl 2 dl] 0 setdash 0 0 0 setrgbcolor } def /LT0 { PL [] 0 1 0 DL } def /LT1 { PL [4 dl 2 dl] 0 0 1 DL } def /LT2 { PL [2 dl 3 dl] 1 0 0 DL } def /LT3 { PL [1 dl 1.5 dl] 1 0 1 DL } def /LT4 { PL [5 dl 2 dl 1 dl 2 dl] 0 1 1 DL } def /LT5 { PL [4 dl 3 dl 1 dl 3 dl] 1 1 0 DL } def /LT6 { PL [2 dl 2 dl 2 dl 4 dl] 0 0 0 DL } def /LT7 { PL [2 dl 2 dl 2 dl 2 dl 2 dl 4 dl] 1 0.3 0 DL } def /LT8 { PL [2 dl 2 dl 2 dl 2 dl 2 dl 2 dl 2 dl 4 dl] 0.5 0.5 0.5 DL } def /M {moveto} def /L {lineto} def /P { stroke [] 0 setdash currentlinewidth 2 div sub moveto 0 currentlinewidth rlineto stroke } def /D { stroke [] 0 setdash 2 copy vpt add moveto hpt neg vpt neg rlineto hpt vpt neg rlineto hpt vpt rlineto hpt neg vpt rlineto closepath stroke P } def /A { stroke [] 0 setdash vpt sub moveto 0 vpt2 rlineto currentpoint stroke moveto hpt neg vpt neg rmoveto hpt2 0 rlineto stroke } def /B { stroke [] 0 setdash 2 copy exch hpt sub exch vpt add moveto 0 vpt2 neg rlineto hpt2 0 rlineto 0 vpt2 rlineto hpt2 neg 0 rlineto closepath stroke P } def /C { stroke [] 0 setdash exch hpt sub exch vpt add moveto hpt2 vpt2 neg rlineto currentpoint stroke moveto hpt2 neg 0 rmoveto hpt2 vpt2 rlineto stroke } def /T { stroke [] 0 setdash 2 copy vpt 1.12 mul add moveto hpt neg vpt -1.62 mul rlineto hpt 2 mul 0 rlineto hpt neg vpt 1.62 mul rlineto closepath stroke P } def /S { 2 copy A C} def end gnudict begin gsave 50 50 translate 0.100 0.100 scale 0 setgray /Courier findfont 70 scalefont setfont newpath LTa 504 246 M 504 2344 L LTb 504 246 M 567 246 L 3453 246 M 3390 246 L 462 246 M (100) Rshow 504 596 M 567 596 L 3453 596 M 3390 596 L 462 596 M (150) Rshow 504 945 M 567 945 L 3453 945 M 3390 945 L 462 945 M (200) Rshow 504 1295 M 567 1295 L 3453 1295 M 3390 1295 L 462 1295 M (250) Rshow 504 1645 M 567 1645 L 3453 1645 M 3390 1645 L 462 1645 M (300) Rshow 504 1994 M 567 1994 L 3453 1994 M 3390 1994 L 462 1994 M (350) Rshow 504 2344 M 567 2344 L 3453 2344 M 3390 2344 L 462 2344 M (400) Rshow 504 246 M 504 309 L 504 2344 M 504 2281 L 504 176 M (0) Cshow 873 246 M 873 309 L 873 2344 M 873 2281 L 873 176 M (100) Cshow 1241 246 M 1241 309 L 1241 2344 M 1241 2281 L 1241 176 M (200) Cshow 1610 246 M 1610 309 L 1610 2344 M 1610 2281 L 1610 176 M (300) Cshow 1979 246 M 1979 309 L 1979 2344 M 1979 2281 L 1979 176 M (400) Cshow 2347 246 M 2347 309 L 2347 2344 M 2347 2281 L 2347 176 M (500) Cshow 2716 246 M 2716 309 L 2716 2344 M 2716 2281 L 2716 176 M (600) Cshow 3084 246 M 3084 309 L 3084 2344 M 3084 2281 L 3084 176 M (700) Cshow 3453 246 M 3453 309 L 3453 2344 M 3453 2281 L 3453 176 M (800) Cshow LTb 504 246 M 3453 246 L 3453 2344 L 504 2344 L 504 246 L 1978 2414 M (Incremental Round Trip Latency) Cshow 2531 -12 M (Message size \(bytes\)) Lshow 228 2169 M (usec) Lshow LT0 LT0 3269 666 M (Raw ATM + AAL3/4 + CS Protocol) Rshow 3311 666 M 3437 666 L 637 554 M 637 554 L 799 694 L 961 791 L 1123 917 L 1448 1099 L 2391 1778 L 3296 2344 L 3353 666 D 637 554 D 799 694 D 961 791 D 1123 917 D 1448 1099 D 2391 1778 D LT1 3269 596 M (Sequenced Packet Protocol) Rshow 3311 596 M 3437 596 L 637 610 M 637 610 L 799 715 L 961 1267 L 1123 1386 L 1448 1582 L 2391 2211 L 2589 2344 L 3353 596 A 637 610 A 799 715 A 961 1267 A 1123 1386 A 1448 1582 A 2391 2211 A LT2 3269 526 M (Datagram Protocol) Rshow 3311 526 M 3437 526 L 622 715 M 622 715 L 784 819 L 946 1302 L 1109 1421 L 1448 1603 L 2391 2253 L 2532 2344 L 3353 526 B 622 715 B 784 819 B 946 1302 B 1109 1421 B 1448 1603 B 2391 2253 B stroke grestore end showpage %%EndDocument endTexFig 47 1793 a(Figure)h(3:)j(Comparativ)n(e)c(pro)q(cess-to-pro)q(cess) h(round-trip)g(latencies)g(using)g(our)f(integrated)f(proto)q(cols.)0 1968 y fh(6.)52 b(Related)19 b(w)n(ork)0 2111 y fj(Similarities)h(b)q(et)o(w)n(een)e (our)f(w)o(ork)f(and)i(that)e(of)h(others)g(exists)g(along)h(t)n(w)o(o)f(dime\ nsions:)25 b(protocol)18 b(structure)0 2168 y(and)13 b(protocol)g(strategy)-5 b(.)19 b(In)13 b(terms)f(of)g(proto)q(col)h(structure,)f(integrated)g(la)o (y)o(er)g(pro)q(cessing)i(w)n(as)e(\014rst)h(prop)q(osed)0 2224 y(b)o(y)22 b (Clark)g(and)h(T)l(ennenhouse)g(as)f(a)g(necessary)g(tec)o(hnique)i(to)e(ac)n (hieve)g(go)q(o)q(d)h(protocol)g(p)q(erformance)f(b)o(y)0 2280 y(eliminating) c(the)e(n)o(um)o(b)r(er)g(of)f(times)i(that)e(data)g(in)i(a)f(message)f(is)i (manipulated)g([7].)22 b(Our)16 b(integrated)f(bu\013er)0 2337 y(managemen)o (t)10 b(strategies)g(are)h(similar)h(to)e(those)g(used)i(in)f(DEC)g(SR)o(C)g (RPC)g(implementation)g([27)o(].)18 b(Our)11 b(o)n(v)o(erall)0 2393 y(approac) n(h)18 b(of)f(in)o(tegrating)g(A)l(TM,)g(SAR,)g(CS,)g(bu\013ering,)h(\015o)o (w)f(con)o(trol,)g(and)h(error)e(reco)o(v)o(ery)h(demonstrates)0 2450 y(that) 12 b(additional)i(functionalit)q(y)f(\(for)e(example,)j(error)e(reco)o(v)o (ery\))g(can)h(b)r(e)g(in)o(tegrated)f(into)g(a)h(single)h(la)o(y)o(er)f(with) 0 2506 y(little)g(o)o(v)n(erhead.)19 b(Proto)q(col)12 b(b)o(ypass)f(has)h(b)r (een)g(prop)q(osed)g(b)r(efore)f(b)o(y)h(the)g fg(x)p fj(-k)o(ernel)h([14)o (])e(dev)o(elop)r(ers,)h(with)g(their)0 2563 y(emphasis)17 b(on)f(the)g fg (factorization)g fj(of)f(common)h(la)o(y)n(ers)g(in)h(di\013erent)e(proto)q (col)h(stac)n(ks.)22 b(F)l(actorization)16 b(implies)0 2619 y(a)f(la)o(y)o (ered)g(implemen)q(tation,)f(though,)h(which)g(con\015icts)h(with)g(the)f(goa\ ls)g(of)g(in)o(tegrated)g(la)o(y)n(er)h(pro)q(cessing.)62 2717 y(In)k(terms)e (of)g(proto)q(col)h(strategy)-5 b(,)19 b(our)g(\015o)n(w)g(con)o(trol)f(and)h (congestion)g(a)o(v)o(oidance)g(sc)o(heme)g(is)h(similar)g(to)952 2844 y(14)p eop %%Page: 15 18 bop 0 42 a fj(TCP's)14 b(in)i(that)f(w)n(e)h(use)f(sliding)j(windo)o(ws,)d (an)g(initial)i(window)e(size)h(of)f(one)g(segmen)o(t,)f(increase)i(the)g(win\ do)o(w)0 98 y(size)i(after)e(ac)o(kno)n(wledgement)h(of)f(the)h(initial)i(win\ dow,)d(and)i(drop)e(the)h(window)g(size)g(bac)o(k)g(to)f(one)h(segmen)o(t)0 154 y(in)g(case)e(of)h(timeout)f(and)h(message)f(retransmission.)935 138 y fd (5)977 154 y fj(There)h(are,)f(ho)o(w)n(ev)o(er,)h(imp)r(ortan)n(t)f(di\013er\ ences.)23 b(First,)0 211 y(our)14 b(initial)j(windo)o(w)e(size)g(is)g(m)o(uc) o(h)g(smaller,)g(reducing)h(the)e(probabilit)q(y)g(of)g(cell)j(loss)d(in)i (case)e(other)g(messages)0 267 y(are)f(b)r(eing)h(received)g(concurrently)g (at)f(the)g(destination.)21 b(Second,)14 b(on)g(successful)h(transmission)f (of)f(the)h(initial)0 324 y(windo)o(w,)21 b(w)n(e)f(switc)o(h)g(immediately)h (to)e(a)h(windo)o(w)g(size)g(statically)h(determined)g(to)e(supp)r(ort)g(tran\ smission)0 380 y(at)e(full)i(bandwidth)g(with)f(lo)o(w)g(probability)g(of)f (bu\013er)h(o)n(v)o(errun)g(instead)h(of)e(h)o(un)o(ting)h(for)f(the)h(righ)o (t)g(windo)o(w)0 437 y(size)f(using)h(slo)o(w)e(start.)22 b(Third,)17 b(w)o (e)g(send)g(sync)o(hronizations)g(at)f(eac)o(h)g(window)g(rather)g(than)g(eac) o(h)h(segmen)o(t,)0 493 y(reducing)g(sync)o(hronization)g(bandwidth)f(o)o(v)o (erhead.)21 b(F)l(ourth,)15 b(our)h(retransmission)g(with)g(exp)q(onential)g (bac)o(k-)0 550 y(o\013)f(e\013ectively)i(in)o(tegrates)e(the)i(mec)o(hanism) f(that)g(TCP)g(implicitly)j(uses)e(when)f(managing)g(congestion)h(o)n(v)o(er) 0 606 y(broadcast-based)j(net)o(w)o(orks)g(suc)o(h)g(as)g(Ethernet)h(and)f (FDDI.)g(Fifth,)h(our)g(bu\013er)f(sc)o(heduling)j(co)q(ordinates)0 663 y(rec\ eption)17 b(of)f(messages)g(on)g(di\013erent)g(sessions)h(so)f(as)g(to)f(redu\ ce)j(the)e(probability)h(of)f(FIF)n(O)h(o)n(v)o(errun,)f(while)0 719 y(TCP)h (con)o(trol)h(is)g(p)r(er-connection)h(only)m(.)28 b(Finally)m(,)18 b(our)g (\015o)o(w)f(and)h(congestion)g(con)o(trol)g(algorithms)g(supp)r(ort)0 775 y (b)r(oth)k(connection-oriented)h(and)g(connectionless)i(comm)n(unication,)h (while)e(TCP)e(is)h(connection-orien)q(ted)0 832 y(only)m(.)62 930 y(Our)15 b (\015o)o(w)g(con)o(trol)f(sc)o(heme)i(can)f(b)q(e)g(con)o(trasted)f(to)g(thos\ e)h(b)r(eing)g(inv)n(estigated)g(in)h(DEC)e(SRC's)g(Autonet-)0 986 y(2)e([29) o(])g(and)h(CMU's)e(V)o(C-Nectar)h([30)o(])g(pro)r(jects.)19 b(In)13 b(AN-2,) g(eac)n(h)g(cell)h(received)f(is)f(ac)o(kno)o(wledged)h(individuall)q(y)0 1043 y(on)j(a)h(link-by-link,)h(p)r(er-connection)g(basis,)f(using)g(a)f(non-stand\ ard)h(piggy-bac)o(k)f(\014eld)i(in)g(the)e(A)m(TM)g(header)0 1099 y(of)e(cell\ s)i(\015o)o(wing)e(in)h(the)g(rev)o(erse)f(direction.)21 b(In)15 b(con)o(tras\ t,)e(V)o(C)h(Nectar)g(uses)g(explicit)j fg(cr)m(e)n(dit)f(c)m(el)r(ls)e fj (to)f(indicate)0 1156 y(the)21 b(existence)h(of)e(reserv)o(ed)h(bu\013ers)g (for)f(a)g(sp)q(eci\014ed)j(n)o(um)o(b)q(er)e(of)f(cells)j(o)n(v)o(er)d(a)h (connection.)37 b(The)21 b(sender)0 1212 y(only)16 b(attempts)f(transmission) i(if)f(it)g(has)g(obtained)g(the)g(required)h(credit)g(from)e(the)h(next)g (no)q(de)h(on)e(the)h(path.)0 1269 y(Our)e(sync)o(hronization)h(cells)h(ha)n (v)o(e)e(function)h(similar)g(to)e(AN-2's)h(ac)n(kno)o(wledgements)f(or)h(V)n (C)g(Nectar's)f(credit)0 1325 y(cells,)25 b(but)d(on)h(an)f(end-to-end)h(basi\ s.)41 b(Link-b)q(y-link)23 b(con)o(trol)f(allows)g(more)f(precise)j(trac)n (king)f(of)e(widely)0 1381 y(v)m(arying)f(tra\016c)f(conditions,)i(but)f(at)f (the)h(exp)q(ense)h(of)e(considerably)i(more)e(sophisticated)i(host)e(in)o (terfaces)0 1438 y(and)c(switc)o(hes.)0 1603 y fh(7.)52 b(Conclusions)0 1746 y fj(The)16 b(high-bandwidth,)i(lo)o(w-latency)e(p)r(erformance)f(of)h(the)g (coming)g(generation)g(of)g(net)n(w)o(orks)f(can)h(b)r(e)g(easily)0 1802 y (sacri\014ced)i(through)f(redundant)f(proto)q(col)i(pro)q(cessing.)26 b(These) 17 b(net)o(w)o(orks,)g(suc)o(h)g(as)g(A)l(TM,)f(require)i(sp)r(eci\014c)0 1858 y(lo)o(w-level)e(supp)r(ort)f(for)g(services)i(suc)o(h)f(as)g(connection)h (managemen)n(t,)e(\015o)o(w)h(con)o(trol,)f(congestion)h(a)o(v)n(oidance,)0 1915 y(and)f(segmen)o(tation)g(and)g(reassem)o(bly)m(.)20 b(Slight,)14 b(but) h(w)o(ell-in)q(tegrated)g(extensions)g(to)g(these)g(base)g(services)h(are)0 1971 y(su\016cient)d(to)g(ensure)h(e\016cient)f(reliable)i(deliv)q(ery)l(.)k (Our)14 b(w)n(ork)f(demonstrates)g(that)f(it)i(is)g(p)q(ossible)h(to)d(integr\ ate)0 2028 y(m)o(ultiple)20 b(proto)q(col)e(la)o(y)o(ers)g(into)g(a)g(relativ) o(ely)h(simple)h(single-la)q(y)n(ered)f(implemen)q(tation)f(that)f(deliv)q (ers)h(b)q(oth)0 2084 y(lo)o(w)d(latency)h(and)f(high)h(throughput.)0 2249 y fh(References)23 2367 y fj([1])21 b(M.)12 b(Accetta,)g(R.)g(Baron,)g(W.)f(Bol\ osky)m(,)h(D.)f(Golub,)i(R.)f(F.)g(Rashid,)h(A.)f(T)l(ev)n(anian)g(Jr.)g(and) g(M.)g(W.)f(Y)l(oung.)93 2424 y(\\Mac)n(h:)20 b(A)14 b(new)g(k)o(ernel)h(foun\ dation)g(for)e(UNIX)i(dev)o(elopment",)e(in)i fg(Pr)n(o)m(c)n(e)n(e)m(dings)g (of)g(the)g(Summer)h(1986)93 2480 y(USENIX)f(Confer)m(enc)m(e)p fj(,)g(July)h (1986,)e(pp.)h(93-113.)23 2570 y([2])21 b(Ram\023)-22 b(on)11 b(C\023)-23 b (aceres.)11 b(\\E\016ciency)i(of)f(A)l(TM)f(Net)o(w)n(orks)h(in)g(T)l(ransp)q (orting)g(Wide-Area)g(Data)f(T)l(ra\016c",)g(T)l(ech-)93 2627 y(nical)17 b (Rep)r(ort)e(TR-91-043,)f(In)o(ternational)h(Computer)g(Science)i(Institute,) f(Berk)o(eley)m(,)f(CA,)f(July)n(,)h(1991.)0 2667 y 780 2 v 52 2724 a fc(5)69 2740 y fi(The)e(drop-back)f(to)h(one)g(segmen)o(t)g(o)q(ccurs)g(implicitly)j (since)e(a)e(message)h(is)g(retransmitted)h(from)f(the)f(\014rst)h(window)f (on)h(error.)952 2844 y fj(15)p eop %%Page: 16 19 bop 23 42 a fj([3])21 b(D.)e(Cohen,)i(G.)e(Finn,)i(R.)e(F)l(elderman,)i(A.)e (DeSchon.)g(\\A)l(TOMIC:)g(A)h(Lo)o(w-Cost,)f(V)l(ery)h(High-Sp)r(eed)93 98 y (LAN",)13 b(Univ)o(ersit)o(y)f(of)g(Southern)h(California,)g(Information)f (Sciences)i(Institute,)f(ISI/RR-92-291,)g(Sept.)93 154 y(1992.)23 248 y([4]) 21 b(E.)c(Co)q(op)q(er,)g(O.)g(Menzilcioglu,)j(R.)d(Sansom)g(and)g(F)l(ran\ \030)-20 b(cois)18 b(Bitz.)f(\\Host)f(Interface)g(Design)i(F)l(or)e(A)m(TM)93 305 y(LANs",)d(in)g fg(Pr)m(o)n(c)n(e)n(e)m(dings)g(of)g(the)h(16th)g(Confer) m(enc)m(e)f(on)g(L)n(o)m(c)n(al)g(Computer)h(Networks")p fj(,)f(IEEE,)f(Oct.) f(1991,)93 361 y(pp.)16 b(247-258.)23 455 y([5])21 b(P)m(.)15 b(Dale.)g(\\A)g (Critique)h(of)f(OSF/1",)g(presen)o(ted)g(at)g(the)h fg(1992)h(D)o(ARP)l(A)g (Softwar)n(e)f(Systems)f(Workshop)p fj(,)93 511 y(Marc)o(h)g(1993.)23 605 y ([6])21 b(Da)o(vid)i(D.)e(Clark,)j(V)l(an)e(Jacobson,)i(John)f(Romk)o(ey)f (and)h(Ho)n(w)o(ard)f(Salw)o(en.)g(\\An)h(analysis)g(of)e(TCP)93 662 y(pro)q (cessing)c(o)n(v)o(erhead".)e fg(IEEE)g(Communic)m(ations)h(Magazine)p fj(,)e (June)j(1989,)c(pp.)i(23-29.)23 756 y([7])21 b(Da)o(vid)f(D.)f(Clark)h(and)g (Da)n(vid)h(L.)e(T)l(ennenhouse.)i(\\Arc)o(hitectural)g(considerations)f(for) f(a)h(new)g(gener-)93 812 y(ation)g(of)e(proto)q(cols",)i(in)g fg(Pr)n(o)n (c)m(e)n(e)m(dings)g(of)g(the)g(1990)h(SIGCOMM)d(Symp)n(osium)i(on)g(Communic) m(ations)93 868 y(A)o(r)n(chite)n(ctur)n(es)c(and)g(Pr)n(oto)n(c)m(ols)p fj (,)f(A)o(CM,)f(Septem)o(b)q(er)i(1990,)d(pp.)j(200-208.)23 962 y([8])21 b(Dig\ ital)k(Equipment)f(Corp)q(oration.)f(\\DECstation)g(5000/200)f(KN02)i(System) g(Module)i(F)l(unctional)93 1019 y(Sp)r(eci\014cation",)16 b(W)l(orkstation)e (Systems)h(Engineering,)i(P)n(alo)f(Alto,)f(CA,)f(Aug.)h(27,)f(1990.)23 1112 y ([9])21 b(Digital)e(Equipment)f(Corp)q(oration.)f(\\TURBOchannel)i(Hardw)o (are)f(Sp)q(eci\014cation",)i(DEC,)d(Ma)n(ynard,)93 1169 y(MA,)e(Ma)n(y)g(199\ 0.)0 1263 y([10])21 b(F)l(ore)15 b(Systems.)g(\\ASX)g(A)l(TM)g(Switc)o(h)h (Arc)o(hitecture)g(Man)n(ual",)f(v)o(ersion)h(2.0.)d(Pittsburgh,)i(P)m(A,)f (1992.)0 1357 y([11])21 b(F)l(ore)c(Systems.)f(\\TCA-100)f(TURBOchannel)j(A)m (TM)e(Computer)g(Interface)g(User's)g(Man)o(ual",)h(v)n(ersion)93 1413 y(1.0.) d(Pittsburgh,)h(P)l(A,)g(1992.)0 1507 y([12])21 b(F)l(ore)13 b(Systems.\\SP)l (ANS:)g(Simple)j(Protocol)e(for)e(A)m(TM)h(Net)n(w)o(ork)g(Signaling",)i(v)n (ersion)f(2.0.)e(Pittsburgh,)93 1563 y(P)m(A,)i(1992.)0 1657 y([13])21 b(Ken) 15 b(Hardwick.)e(\\HIPPI)i(W)l(orld)f({)g(The)h(Switc)o(h)f(is)h(the)f(Net)o (w)o(ork",)f(in)i fg(Pr)n(o)m(c)n(e)n(e)m(dings)g(of)g(the)h(37th)g(IEEE)93 1714 y(Computer)h(So)n(ciety)f(International)f(Confer)m(enc)m(e)p fj(,)g(F)l (ebruary)g(1992.)0 1807 y([14])21 b(N.C.)14 b(Hutc)o(hinson,)i(L.L.)e(P)o(ete\ rson,)g(M.B.)g(Abb)r(ott,)f(and)i(S.)g(O'Malley)m(.)f(\\RPC)h(in)g(the)g(x-Ke\ rnel:)21 b(Ev)m(al-)93 1864 y(uating)f(New)g(Design)g(T)l(ec)o(hniques",)h (in)g fg(Pr)m(o)n(c)n(e)m(e)n(dings)e(of)i(the)f(12th)h(A)o(CM)e(Symp)n(osium) i(on)f(Op)n(er)m(ating)93 1920 y(System)c(Principles)p fj(,)e(Dec.)g(1989,)g (pp.)h(91-101.)0 2014 y([15])21 b(Da)o(vid)15 b(B.)g(Johnson.)f(\\The)h(P)o (eregrine)g(High-Performance)f(RPC)h(System",)f(in)h fg(Softwar)n(e)h(-)g(Pr) n(actic)m(e)g(&)93 2071 y(Exp)n(erienc)m(e)p fj(,)f(F)l(eb.)g(1993.)0 2164 y ([16])21 b(Gerry)15 b(Kane.)g(\\MIPS)h(Risc)g(Arc)o(hitecture",)f(Pren)o(tice) h(Hall,)g(Englew)o(o)q(o)q(d)f(Cli\013s,)h(NJ,)f(1987.)0 2258 y([17])21 b(J.) c(Ka)o(y)f(and)h(J.)f(P)o(asquale.)g(\\Measuremen)o(t,)g(Analysis,)h(and)g (Impro)o(v)n(ement)f(of)g(UDP/IP)g(Throughput)93 2315 y(for)f(the)g(DECstatio\ n)f(5000",)f(T)m(ec)o(h.)i(Rep)r(ort,)f(Computer)g(Systems)h(Lab,)g(Dept.)g (of)f(Computer)h(Science)93 2371 y(and)h(Engineering,)g(Univ.)g(California)g (at)e(San)i(Diego,)f(Jan)o(uary)g(1993.)0 2465 y([18])21 b(B.W.)16 b(Lampson.) f(\\Hints)g(for)h(Computer)f(System)h(Design",)g(in)h fg(9th)g(A)o(CM)f(Symp) n(osium)h(on)g(Op)n(er)m(ating)93 2521 y(System)f(Principles)p fj(,)e(Octob)q (er)h(1983,)f(pp.)h(33-48.)0 2615 y([19])21 b(Sam)o(uel)f(J.)f(Le\017er,)h (Marshall)f(K.)g(McKusick,)h(Mic)o(hael)g(J.)e(Karels)i(and)f(John)h(S.)f(Qua\ terman.)f(\\The)93 2672 y(Design)h(and)f(Implementation)g(of)f(the)h(4.3BSD)f (UNIX)i(Op)q(erating)f(System",)g(Addison-W)m(esley)h(Pub.)93 2728 y(Co.,)14 b (Reading,)i(MA,)f(1989.)952 2844 y(16)p eop %%Page: 17 20 bop 0 42 a fj([20])21 b(C.)c(Maeda)g(and)h(B.)f(Bershad.)h(\\Proto)o(col)g (Service)h(Decomp)q(osition)f(for)f(High-Performance)g(In)o(ternet-)93 98 y (w)o(orking",)j(Carnegie)f(Mellon)i(Univ)o(ersit)o(y)f(Sc)o(ho)q(ol)g(of)e (Computer)h(Science)j(T)l(ec)o(hnical)f(Rep)r(ort)d(CMU-)93 154 y(CS-93-131,) c(Marc)n(h)i(1993.)0 248 y([21])21 b(R.M.)d(Metcalfe)g(and)g(D.R.)f(Boggs.)g (\\Ethernet:)25 b(Distributed)19 b(pac)o(k)o(et)f(switc)o(hing)h(for)e(lo)r (cal)h(computer)93 305 y(net)o(w)o(orks",)c(in)i fg(Communic)m(ations)g(of)h (the)f(A)o(CM)p fj(,)e(July)i(1976,)e(pp.)h(395-404.)0 399 y([22])21 b(S.E.)c (Minzer.)h(\\Broadband)g(ISDN)g(and)g(async)o(hronous)f(transfer)g(mo)q(de)h (\(A)l(TM\)",)e fg(IEEE)i(Communi-)93 455 y(c)n(ations)e(Magazine)p fj(,)e (Sept.)h(1989,)f(pp.)h(17-24.)0 549 y([23])21 b(John)d(K.)e(Ousterhout.)h(\\W\ h)n(y)g(aren't)f(op)q(erating)h(systems)f(getting)h(faster)f(as)g(fast)g(as)h (hardw)n(are?",)g(in)93 605 y fg(Pr)n(o)n(c)n(e)m(e)n(dings)e(of)i(the)f(Summ\ er)h(1990)g(USENIX)e(Confer)m(enc)m(e)p fj(,)f(June)j(1990,)c(pp.)i(247-256.) 0 699 y([24])21 b(J.B.)15 b(P)o(ostel,)f(editor.)h(T)l(ransmission)h(Con)n (trol)f(Protocol.)g(In)o(ternet)g(Request)h(for)e(Commen)n(ts)h(RF)o(C)f(793,) 93 756 y(Septemb)q(er)h(1981.)0 849 y([25])21 b(R.)16 b(Rashid,)h(A.)e(T)l (ev)m(anian,)h(M.)f(Y)l(oung,)h(D.)f(Golub,)g(R.)h(Baron,)f(D.)g(Blac)o(k,)g (W.)g(Bolosky)h(and)g(J.)f(Chew.)93 906 y(\\Mac)n(hine-Indep)5 b(endent)18 b (Virtual)h(Memory)f(Managemen)n(t)h(for)f(P)n(aged)h(Unipro)r(cessor)f(and)h (Multipro-)93 962 y(cessor)c(Arc)o(hitectures",)g(in)g fg(Pr)n(o)n(c)m(e)n (e)m(dings)g(of)h(the)g(2nd)g(Symp)n(osium)g(on)f(A)o(r)n(chite)n(ctur)n(al)g (Supp)n(ort)i(for)f(Pr)n(o-)93 1019 y(gr)n(amming)g(L)n(anguages)g(and)g(Op)n (er)m(ating)g(Systems)p fj(,)e(A)o(CM,)g(Octob)r(er)h(1987.)0 1112 y([26])21 b (F.E.)16 b(Ross.)g(\\FDDI)f(|)i(An)f(Ov)o(erview",)h(in)g fg(Pr)n(o)n(c)m(e)n (e)m(dings)g(of)g(the)h(COMPCON)d(Spring)i(87)p fj(,)g(San)f(F)l(ran-)93 1169 y(cisco,)g(CA,)f(1987,)e(pp.)j(434-440.)0 1263 y([27])21 b(Michael)14 b(D.)g (Sc)o(hro)q(eder)g(and)g(Michael)g(Burro)o(ws.)f(\\P)o(erformance)g(of)h(Fire\ \015y)g(RPC",)g(in)g fg(A)o(CM)g(T)l(r)n(ansac-)93 1319 y(tions)i(on)g(Comput\ er)h(Systems)p fj(,)d(A)o(CM,)g(F)l(eb.)h(1990,)f(pp.)h(1-17.)0 1413 y([28]) 21 b(S.)12 b(Scott,)f(L.)h(Zhang)f(and)h(D.)e(Clark.)h(\\Some)h(Observ)n(atio\ ns)f(on)h(the)f(Dynamics)h(of)f(a)g(Congestion)h(Con)n(trol)93 1469 y(Algorit\ hm",)j(in)h fg(A)o(CM)g(Computer)h(Communic)m(ation)f(R)n(eview)p fj(,)f(v)o (ol.)g(20,)f(no.)h(5,)f(Octob)r(er)h(1990.)0 1563 y([29])21 b(M.)13 b(Goguen.) f(\\AN2:)18 b(A)13 b(Self-Con\014guring)i(Lo)q(cal)f(A)l(TM)f(Net)n(w)o(ork",) f(presentation)g(at)h(Carnegie)g(Mellon)93 1620 y(Universit)o(y)l(,)i(No)o (v.)f(5,)h(1992.)0 1714 y([30])21 b(P)o(eter)15 b(Steenkiste,)h(p)q(ersonal)g (comm)n(unication.)0 1807 y([31])21 b(A.)16 b(T)l(anenbaum.)f(\\Net)o(w)n(ork) h(Protocols",)h(in)g fg(A)n(CM)g(Computing)g(Surveys)p fj(,)f(v)n(ol.)g(13,)g (no.)g(4,)g(Dec.)f(1981,)93 1864 y(pp.)h(453-489.)0 1958 y([32])21 b(C.)13 b (Thekk)m(ath)g(and)g(H.)f(Levy)m(.)g(\\Limits)i(to)e(Lo)o(w-Latency)h(Comm)n (unication)h(on)f(High-Sp)s(eed)g(Net)o(w)n(orks",)93 2014 y(Universit)o(y)i (of)f(W)l(ashington)h(Dept.)f(of)g(Computer)g(Science)j(and)d(Engineering)j (T)l(ec)o(hnical)f(Rep)r(ort)e(UW-)93 2071 y(TR-91-06-01,)g(June)j(1991.)0 2164 y([33])k(A.W)l(olman,)d(G.)f(V)m(o)q(elk)o(er)h(and)g(C.)f(Thekk)n(ath.) g(\\Latency)h(Analysis)h(of)e(TCP)h(on)g(an)f(A)m(TM)g(Net)o(w)n(ork",)93 2221 y(Universit)o(y)e(of)f(W)l(ashington)h(Dept.)f(of)g(Computer)g(Science)j(and) d(Engineering)j(T)l(ec)o(hnical)f(Rep)r(ort)e(UW-)93 2277 y(TR-93-03-03,)g (Marc)n(h)i(1993.)952 2844 y(17)p eop %%Trailer end %%EOF