forked from I2P_Developers/i2p.i2p
6340 lines
187 KiB
XML
6340 lines
187 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<leo_file>
|
|
<leo_header file_format="2" tnodes="0" max_tnode_index="35" clone_windows="0"/>
|
|
<globals body_outline_ratio="0.34387755102">
|
|
<global_window_position top="157" left="189" height="709" width="980"/>
|
|
<global_log_window_position top="0" left="0" height="0" width="0"/>
|
|
</globals>
|
|
<preferences>
|
|
</preferences>
|
|
<find_panel_settings>
|
|
<find_string></find_string>
|
|
<change_string></change_string>
|
|
</find_panel_settings>
|
|
<vnodes>
|
|
<v t="aum.20040802153926" a="E"><vh>Kademlia</vh>
|
|
<v t="aum.20040802154007" a="E" tnodeList="aum.20040802154007,aum.20040805131021.1,aum.20040802154945,aum.20040803013638,aum.20040803132528,aum.20040802154241,aum.20040811111244,aum.20040811111244.1,aum.20040807013038,aum.20040804122542,aum.20040813204308,aum.20040804122542.1,aum.20040804123117,aum.20040804123350,aum.20040804123526,aum.20040805123632,aum.20040804132551,aum.20040812152603,aum.20040807052750,aum.20040807053418,aum.20040807114327,aum.20040813204858,aum.20040804130347,aum.20040807013038.1,aum.20040807013038.2,aum.20040803005037,aum.20040803005037.1,aum.20040803005037.2,aum.20040803005037.3,aum.20040803005037.4,aum.20040803005309,aum.20040803134838,aum.20040803134838.1,aum.20040803010321,aum.20040803010321.1,aum.20040803134418,aum.20040803140601,aum.20040808133629,aum.20040808133941,aum.20040802154241.1,aum.20040802154241.2,aum.20040802164545,aum.20040808160207,aum.20040804001454,aum.20040813015858,aum.20040802165054,aum.20040803013434,aum.20040803013434.1,aum.20040803013638.1,aum.20040805182736,aum.20040802155814.2,aum.20040802155814.3,aum.20040802160335,aum.20040802160828,aum.20040802160828.1,aum.20040802160828.2,aum.20040805040409,aum.20040802184641,aum.20040804014643,aum.20040803143007,aum.20040803143007.1,aum.20040803143007.2,aum.20040804213616,aum.20040804220932,aum.20040807013038.3,aum.20040805012557,aum.20040805154232,aum.20040805012557.1,aum.20040806144708,aum.20040806144812,aum.20040806144829,aum.20040805001742,aum.20040805001926,aum.20040805004949,aum.20040805013630,aum.20040805001258,aum.20040805030707,aum.20040805185902,aum.20040805013903,aum.20040805013957,aum.20040807013038.4,aum.20040805014353,aum.20040805154253,aum.20040805014353.1,aum.20040805014900,aum.20040805014900.1,aum.20040805032351,aum.20040807013038.5,aum.20040806223556,aum.20040807033258,aum.20040807033258.1,aum.20040807033258.2,aum.20040807033818,aum.20040807033818.1,aum.20040807034729,aum.20040806223857,aum.20040807004327,aum.20040807004327.1,aum.20040807004327.5,aum.20040807004327.6,aum.20040807004327.11,aum.20040807004327.12,aum.20040811163127,aum.20040811221318,aum.20040807004327.13,aum.20040807004327.14,aum.20040807004327.15,aum.20040807004434,aum.20040807013835,aum.20040811235333,aum.20040807013944,aum.20040807004327.2,aum.20040807004327.3,aum.20040807004327.4,aum.20040807004327.7,aum.20040807004327.8,aum.20040807004327.9,aum.20040807004327.10,aum.20040807014538,aum.20040807044007,aum.20040805130159,aum.20040805143215,aum.20040805154306,aum.20040805130514,aum.20040805140416,aum.20040806234241,aum.20040807044007.1,aum.20040805185215,aum.20040806002319,aum.20040806132808,aum.20040805140632,aum.20040805141509,aum.20040811221628,aum.20040811222934,aum.20040807003220,aum.20040807013411,aum.20040813200853,aum.20040807013038.6,aum.20040805153146,aum.20040805154321,aum.20040808153427,aum.20040808163651,aum.20040816141128,aum.20040816141128.1,aum.20040816014757,aum.20040807013038.7,aum.20040805153315,aum.20040805154344,aum.20040808134739,aum.20040808134739.1,aum.20040816133040,aum.20040816135222,aum.20040816135222.1,aum.20040808140937,aum.20040808135302,aum.20040808140937.1,aum.20040808140937.2,aum.20040815164410,aum.20040815164410.1,aum.20040815164410.2,aum.20040815164410.3,aum.20040815164410.4,aum.20040815164410.7,aum.20040815164410.8,aum.20040815164410.5,aum.20040813232532,aum.20040813232532.1,aum.20040813233036,aum.20040813234214,aum.20040813232532.2,aum.20040813232532.3,aum.20040814004432,aum.20040814001156,aum.20040814001156.1,aum.20040814001912,aum.20040814001456,aum.20040814001522,aum.20040814001456.2,aum.20040814002236,aum.20040814103533,aum.20040814131117,aum.20040815170456,aum.20040814003559,aum.20040814002411,aum.20040814002411.1,aum.20040807013038.8,aum.20040802161601,aum.20040813204308.1,aum.20040802184641.1,aum.20040808203152,aum.20040805001449,aum.20040802161601.1,aum.20040802161601.2,aum.20040802161601.3,aum.20040802161601.4,aum.20040803015148,aum.20040802161601.5,aum.20040802161601.6,aum.20040803131111.2,aum.20040803201812.1,aum.20040815170327,aum.20040805140236,aum.20040805153555,aum.20040805153555.1,aum.20040803134102,aum.20040803131210,aum.20040803131210.1,aum.20040803131210.2,aum.20040803131210.3,aum.20040804205057,aum.20040804132053,aum.20040804200916,aum.20040803144052,aum.20040803163704,aum.20040804013647,aum.20040804014115.1,aum.20040808142950,aum.20040805040237,aum.20040803163704.1,aum.20040813234004,aum.20040813234004.1,aum.20040803142127,aum.20040803142127.1,aum.20040803142127.2,aum.20040803193605,aum.20040803200819,aum.20040804130722,aum.20040803132156,aum.20040814110540,aum.20040814110755,aum.20040805031135,aum.20040805031135.1,aum.20040808133629.1,aum.20040803132528.1,aum.20040810224601,aum.20040814112703,aum.20040814112703.1,aum.20040814120624,aum.20040813212609,aum.20040814015747,aum.20040813211551,aum.20040803141131,aum.20040812110124"><vh>@file stasher.py</vh>
|
|
<v t="aum.20040805131021.1" a="M"><vh>explanatory comments</vh></v>
|
|
<v t="aum.20040802154945"><vh>imports</vh></v>
|
|
<v t="aum.20040803013638"><vh>constants</vh></v>
|
|
<v t="aum.20040803132528"><vh>globals</vh></v>
|
|
<v t="aum.20040802154241"><vh>Exceptions</vh></v>
|
|
<v t="aum.20040811111244" a="E"><vh>Mixins</vh>
|
|
<v t="aum.20040811111244.1"><vh>class KBase</vh></v>
|
|
</v>
|
|
<v t="aum.20040807013038" a="E"><vh>Main Engine</vh>
|
|
<v t="aum.20040804122542" a="E"><vh>class KCore</vh>
|
|
<v t="aum.20040813204308"><vh>attributes</vh></v>
|
|
<v t="aum.20040804122542.1"><vh>__init__</vh></v>
|
|
<v t="aum.20040804123117"><vh>subscribe</vh></v>
|
|
<v t="aum.20040804123350"><vh>unsubscribe</vh></v>
|
|
<v t="aum.20040804123526"><vh>threadRxPackets</vh></v>
|
|
<v t="aum.20040805123632"><vh>threadHousekeeping</vh></v>
|
|
<v t="aum.20040804132551"><vh>nodeWhichOwnsSock</vh></v>
|
|
<v t="aum.20040812152603"><vh>cycle</vh></v>
|
|
<v t="aum.20040807052750"><vh>run</vh></v>
|
|
<v t="aum.20040807053418"><vh>stop</vh></v>
|
|
<v t="aum.20040807114327"><vh>runClient</vh></v>
|
|
<v t="aum.20040813204858"><vh>select</vh></v>
|
|
</v>
|
|
<v t="aum.20040804130347"><vh>create instance</vh></v>
|
|
</v>
|
|
<v t="aum.20040807013038.1" a="E"><vh>Basic Classes</vh>
|
|
<v t="aum.20040807013038.2" a="E"><vh>Node-local Storage</vh>
|
|
<v t="aum.20040803005037" a="E"><vh>class KStorageBase</vh>
|
|
<v t="aum.20040803005037.1"><vh>__init__</vh></v>
|
|
<v t="aum.20040803005037.2"><vh>putRefs</vh></v>
|
|
<v t="aum.20040803005037.3"><vh>getRefs</vh></v>
|
|
<v t="aum.20040803005037.4"><vh>putKey</vh></v>
|
|
<v t="aum.20040803005309"><vh>getKey</vh></v>
|
|
<v t="aum.20040803134838" a="E"><vh>private methods</vh>
|
|
<v t="aum.20040803134838.1"><vh>_expandRefsList</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040803010321" a="E"><vh>class KStorageFile</vh>
|
|
<v t="aum.20040803010321.1"><vh>__init__</vh></v>
|
|
<v t="aum.20040803134418"><vh>putRefs</vh></v>
|
|
<v t="aum.20040803140601"><vh>getRefs</vh></v>
|
|
<v t="aum.20040808133629"><vh>putKey</vh></v>
|
|
<v t="aum.20040808133941"><vh>getKey</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040802154241.1" a="E"><vh>class KHash</vh>
|
|
<v t="aum.20040802154241.2"><vh>__init__</vh></v>
|
|
<v t="aum.20040802164545"><vh>__str__</vh></v>
|
|
<v t="aum.20040808160207"><vh>asHex</vh></v>
|
|
<v t="aum.20040804001454"><vh>distance</vh></v>
|
|
<v t="aum.20040813015858"><vh>rawdistance</vh></v>
|
|
<v t="aum.20040802165054"><vh>operators</vh></v>
|
|
</v>
|
|
<v t="aum.20040803013434"><vh>class KBucket</vh>
|
|
<v t="aum.20040803013434.1"><vh>__init__</vh></v>
|
|
<v t="aum.20040803013638.1"><vh>justSeenPeer</vh></v>
|
|
<v t="aum.20040805182736"><vh>__iter__</vh></v>
|
|
</v>
|
|
<v t="aum.20040802155814.2" a="E"><vh>class KPeer</vh>
|
|
<v t="aum.20040802155814.3"><vh>__init__</vh></v>
|
|
<v t="aum.20040802160335"><vh>send_ping</vh></v>
|
|
<v t="aum.20040802160828"><vh>send_store</vh></v>
|
|
<v t="aum.20040802160828.1"><vh>send_findNode</vh></v>
|
|
<v t="aum.20040802160828.2"><vh>send_findData</vh></v>
|
|
<v t="aum.20040805040409"><vh>send_reply</vh></v>
|
|
<v t="aum.20040802184641"><vh>send_raw</vh></v>
|
|
<v t="aum.20040804014643"><vh>justSeen</vh></v>
|
|
<v t="aum.20040803143007" a="E"><vh>lowlevel</vh>
|
|
<v t="aum.20040803143007.1"><vh>__str__</vh></v>
|
|
<v t="aum.20040803143007.2"><vh>__repr__</vh></v>
|
|
<v t="aum.20040804213616"><vh>__eq__</vh></v>
|
|
<v t="aum.20040804220932"><vh>__ne__</vh></v>
|
|
</v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040807013038.3" a="E"><vh>RPC Classes</vh>
|
|
<v t="aum.20040805012557" a="E"><vh>class KRpc</vh>
|
|
<v t="aum.20040805154232"><vh>attribs</vh></v>
|
|
<v t="aum.20040805012557.1"><vh>__init__</vh></v>
|
|
<v t="aum.20040806144708"><vh>__del__</vh></v>
|
|
<v t="aum.20040806144812"><vh>__str__</vh></v>
|
|
<v t="aum.20040806144829"><vh>__repr__</vh></v>
|
|
<v t="aum.20040805001742"><vh>bindPeerReply</vh></v>
|
|
<v t="aum.20040805001926"><vh>unbindPeerReply</vh></v>
|
|
<v t="aum.20040805004949"><vh>unbindAll</vh></v>
|
|
<v t="aum.20040805013630"><vh>start</vh></v>
|
|
<v t="aum.20040805001258"><vh>execute</vh></v>
|
|
<v t="aum.20040805030707"><vh>terminate</vh></v>
|
|
<v t="aum.20040805185902"><vh>returnValue</vh></v>
|
|
<v t="aum.20040805013903"><vh>on_reply</vh></v>
|
|
<v t="aum.20040805013957"><vh>on_tick</vh></v>
|
|
</v>
|
|
<v t="aum.20040807013038.4" a="E"><vh>PING</vh>
|
|
<v t="aum.20040805014353" a="E"><vh>class KRpcPing</vh>
|
|
<v t="aum.20040805154253"><vh>attribs</vh></v>
|
|
<v t="aum.20040805014353.1"><vh>__init__</vh></v>
|
|
<v t="aum.20040805014900"><vh>start</vh></v>
|
|
<v t="aum.20040805014900.1"><vh>on_reply</vh></v>
|
|
<v t="aum.20040805032351"><vh>on_tick</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040807013038.5" a="E"><vh>FIND_NODE</vh>
|
|
<v t="aum.20040806223556" a="E"><vh>class KPeerQueryRecord</vh>
|
|
<v t="aum.20040807033258"><vh>__init__</vh></v>
|
|
<v t="aum.20040807033258.1"><vh>hasTimedOut</vh></v>
|
|
<v t="aum.20040807033258.2"><vh>__cmp__</vh></v>
|
|
<v t="aum.20040807033818"><vh>__lt__ etc</vh></v>
|
|
<v t="aum.20040807033818.1"><vh>isCloserThanAllOf</vh></v>
|
|
<v t="aum.20040807034729"><vh>isCloserThanOneOf</vh></v>
|
|
</v>
|
|
<v t="aum.20040806223857" a="E"><vh>class KPeerQueryTable</vh>
|
|
<v t="aum.20040807004327"><vh>__init__</vh></v>
|
|
<v t="aum.20040807004327.1"><vh>setlist</vh></v>
|
|
<v t="aum.20040807004327.5"><vh>getExpired</vh></v>
|
|
<v t="aum.20040807004327.6"><vh>purgeExpired</vh></v>
|
|
<v t="aum.20040807004327.11"><vh>sort</vh></v>
|
|
<v t="aum.20040807004327.12"><vh>select</vh></v>
|
|
<v t="aum.20040811163127"><vh>count</vh></v>
|
|
<v t="aum.20040811221318"><vh>changeState</vh></v>
|
|
<v t="aum.20040807004327.13"><vh>filter</vh></v>
|
|
<v t="aum.20040807004327.14"><vh>purge</vh></v>
|
|
<v t="aum.20040807004327.15"><vh>chooseN</vh></v>
|
|
<v t="aum.20040807004434"><vh>__str__</vh></v>
|
|
<v t="aum.20040807013835"><vh>newtable</vh></v>
|
|
<v t="aum.20040811235333"><vh>dump</vh></v>
|
|
<v t="aum.20040807013944" a="E"><vh>list-like methods</vh>
|
|
<v t="aum.20040807004327.2"><vh>extend</vh></v>
|
|
<v t="aum.20040807004327.3"><vh>append</vh></v>
|
|
<v t="aum.20040807004327.4"><vh>remove</vh></v>
|
|
<v t="aum.20040807004327.7"><vh>__getitem__</vh></v>
|
|
<v t="aum.20040807004327.8"><vh>__len__</vh></v>
|
|
<v t="aum.20040807004327.9"><vh>__getslice__</vh></v>
|
|
<v t="aum.20040807004327.10"><vh>__iter__</vh></v>
|
|
<v t="aum.20040807014538"><vh>__add__</vh></v>
|
|
<v t="aum.20040807044007"><vh>__contains__</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040805130159" a="E"><vh>class KRpcFindNode</vh>
|
|
<v t="aum.20040805143215"><vh>spec info comments</vh></v>
|
|
<v t="aum.20040805154306"><vh>attribs</vh></v>
|
|
<v t="aum.20040805130514"><vh>__init__</vh></v>
|
|
<v t="aum.20040805140416" a="E"><vh>start</vh></v>
|
|
<v t="aum.20040806234241"><vh>sendSomeQueries</vh></v>
|
|
<v t="aum.20040807044007.1"><vh>sendOneQuery</vh></v>
|
|
<v t="aum.20040805185215"><vh>findClosestPeersInitial</vh></v>
|
|
<v t="aum.20040806002319"><vh>addPeerIfCloser</vh></v>
|
|
<v t="aum.20040806132808"><vh>isCloserThanQueried</vh></v>
|
|
<v t="aum.20040805140632"><vh>on_reply</vh></v>
|
|
<v t="aum.20040805141509"><vh>on_tick</vh></v>
|
|
<v t="aum.20040811221628"><vh>checkEndOfRound</vh></v>
|
|
<v t="aum.20040811222934"><vh>gotAnyCloser</vh></v>
|
|
<v t="aum.20040807003220"><vh>returnTheBestWeGot</vh></v>
|
|
<v t="aum.20040807013411"><vh>returnValue</vh></v>
|
|
<v t="aum.20040813200853"><vh>reportStats</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040807013038.6" a="E"><vh>FIND_DATA</vh>
|
|
<v t="aum.20040805153146" a="E"><vh>class KRpcFindData</vh>
|
|
<v t="aum.20040805154321"><vh>attribs</vh></v>
|
|
<v t="aum.20040808153427"><vh>start</vh></v>
|
|
<v t="aum.20040808163651"><vh>on_reply</vh></v>
|
|
<v t="aum.20040816141128"><vh>on_gotValue</vh></v>
|
|
<v t="aum.20040816141128.1"><vh>on_gotChunk</vh></v>
|
|
<v t="aum.20040816014757"><vh>returnValue</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040807013038.7" a="E"><vh>STORE</vh>
|
|
<v t="aum.20040805153315" a="E"><vh>class KRpcStore</vh>
|
|
<v t="aum.20040805154344"><vh>attribs</vh></v>
|
|
<v t="aum.20040808134739"><vh>__init__</vh></v>
|
|
<v t="aum.20040808134739.1"><vh>start</vh></v>
|
|
<v t="aum.20040816133040"><vh>storeSplit</vh></v>
|
|
<v t="aum.20040816135222"><vh>on_doneChunkManifest</vh></v>
|
|
<v t="aum.20040816135222.1"><vh>on_doneChunk</vh></v>
|
|
<v t="aum.20040808140937"><vh>returnValue</vh></v>
|
|
<v t="aum.20040808135302"><vh>on_doneFindNode</vh></v>
|
|
<v t="aum.20040808140937.1"><vh>on_reply</vh></v>
|
|
<v t="aum.20040808140937.2"><vh>on_tick</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040815164410" a="E"><vh>PINGALL</vh>
|
|
<v t="aum.20040815164410.1" a="E"><vh>class KRpcPingAll</vh>
|
|
<v t="aum.20040815164410.2"><vh>attribs</vh></v>
|
|
<v t="aum.20040815164410.3"><vh>__init__</vh></v>
|
|
<v t="aum.20040815164410.4"><vh>start</vh></v>
|
|
<v t="aum.20040815164410.7"><vh>on_reply</vh></v>
|
|
<v t="aum.20040815164410.8"><vh>on_tick</vh></v>
|
|
<v t="aum.20040815164410.5"><vh>returnValue</vh></v>
|
|
</v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040813232532" a="E"><vh>Node Socket Server</vh>
|
|
<v t="aum.20040813232532.1" a="E"><vh>class KNodeServer</vh>
|
|
<v t="aum.20040813233036"><vh>__init__</vh></v>
|
|
<v t="aum.20040813234214"><vh>serve_forever</vh></v>
|
|
</v>
|
|
<v t="aum.20040813232532.2" a="E"><vh>class KNodeReqHandler</vh>
|
|
<v t="aum.20040813232532.3"><vh>handle</vh></v>
|
|
<v t="aum.20040814004432"><vh>finish</vh></v>
|
|
</v>
|
|
<v t="aum.20040814001156" a="E"><vh>class KNodeClient</vh>
|
|
<v t="aum.20040814001156.1"><vh>__init__</vh></v>
|
|
<v t="aum.20040814001912"><vh>hello</vh></v>
|
|
<v t="aum.20040814001456"><vh>connect</vh></v>
|
|
<v t="aum.20040814001522"><vh>close</vh></v>
|
|
<v t="aum.20040814001456.2"><vh>get</vh></v>
|
|
<v t="aum.20040814002236"><vh>put</vh></v>
|
|
<v t="aum.20040814103533"><vh>addref</vh></v>
|
|
<v t="aum.20040814131117"><vh>getref</vh></v>
|
|
<v t="aum.20040815170456"><vh>pingall</vh></v>
|
|
<v t="aum.20040814003559"><vh>kill</vh></v>
|
|
<v t="aum.20040814002411"><vh>__getitem__</vh></v>
|
|
<v t="aum.20040814002411.1"><vh>__setitem__</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040807013038.8" a="E"><vh>NODE</vh>
|
|
<v t="aum.20040802161601" a="E"><vh>class KNode</vh>
|
|
<v t="aum.20040813204308.1"><vh>attributes</vh></v>
|
|
<v t="aum.20040802184641.1"><vh>__init__</vh></v>
|
|
<v t="aum.20040808203152"><vh>__del__</vh></v>
|
|
<v t="aum.20040805001449" a="E"><vh>application-level</vh>
|
|
<v t="aum.20040802161601.1"><vh>start</vh></v>
|
|
<v t="aum.20040802161601.2"><vh>stop</vh></v>
|
|
<v t="aum.20040802161601.3"><vh>get</vh></v>
|
|
<v t="aum.20040802161601.4"><vh>put</vh></v>
|
|
<v t="aum.20040803015148"><vh>addref</vh></v>
|
|
<v t="aum.20040802161601.5"><vh>__getitem__</vh></v>
|
|
<v t="aum.20040802161601.6"><vh>__setitem__</vh></v>
|
|
</v>
|
|
<v t="aum.20040803131111.2" a="E"><vh>peer/rpc methods</vh>
|
|
<v t="aum.20040803201812.1"><vh>_ping</vh></v>
|
|
<v t="aum.20040815170327"><vh>_pingall</vh></v>
|
|
<v t="aum.20040805140236"><vh>_findnode</vh></v>
|
|
<v t="aum.20040805153555"><vh>_finddata</vh></v>
|
|
<v t="aum.20040805153555.1"><vh>_store</vh></v>
|
|
<v t="aum.20040803134102"><vh>_findPeer</vh></v>
|
|
</v>
|
|
<v t="aum.20040803131210" a="E"><vh>comms methods</vh>
|
|
<v t="aum.20040803131210.1"><vh>_sendRaw</vh></v>
|
|
</v>
|
|
<v t="aum.20040803131210.2" a="E"><vh>engine</vh>
|
|
<v t="aum.20040803131210.3"><vh>_threadRx</vh></v>
|
|
<v t="aum.20040804205057"><vh>_doChug</vh></v>
|
|
<v t="aum.20040804132053"><vh>_doRx</vh></v>
|
|
<v t="aum.20040804200916"><vh>_doHousekeeping</vh></v>
|
|
</v>
|
|
<v t="aum.20040803144052" a="E"><vh>event handling</vh>
|
|
<v t="aum.20040803163704"><vh>_on_ping</vh></v>
|
|
<v t="aum.20040804013647"><vh>_on_findNode</vh></v>
|
|
<v t="aum.20040804014115.1"><vh>_on_findData</vh></v>
|
|
<v t="aum.20040808142950"><vh>_on_store</vh></v>
|
|
<v t="aum.20040805040237"><vh>_on_reply</vh></v>
|
|
<v t="aum.20040803163704.1"><vh>_on_unknown</vh></v>
|
|
</v>
|
|
<v t="aum.20040813234004" a="E"><vh>Socket Client Server</vh>
|
|
<v t="aum.20040813234004.1"><vh>serve</vh></v>
|
|
</v>
|
|
<v t="aum.20040803142127" a="E"><vh>lowlevel stuff</vh>
|
|
<v t="aum.20040803142127.1"><vh>__str__</vh></v>
|
|
<v t="aum.20040803142127.2"><vh>__repr__</vh></v>
|
|
<v t="aum.20040803193605"><vh>_msgIdAlloc</vh></v>
|
|
<v t="aum.20040803200819"><vh>_normalisePeer</vh></v>
|
|
<v t="aum.20040804130722"><vh>__del__</vh></v>
|
|
</v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040803132156" a="E"><vh>funcs</vh>
|
|
<v t="aum.20040814110540"><vh>userI2PDir</vh></v>
|
|
<v t="aum.20040814110755"><vh>nodePidfile</vh></v>
|
|
<v t="aum.20040805031135"><vh>messageEncode</vh></v>
|
|
<v t="aum.20040805031135.1"><vh>messageDecode</vh></v>
|
|
<v t="aum.20040808133629.1"><vh>shahash</vh></v>
|
|
<v t="aum.20040803132528.1"><vh>log</vh></v>
|
|
<v t="aum.20040810224601"><vh>logexc</vh></v>
|
|
<v t="aum.20040814112703"><vh>spawnproc</vh></v>
|
|
<v t="aum.20040814112703.1"><vh>killproc</vh></v>
|
|
<v t="aum.20040814120624"><vh>i2psocket</vh></v>
|
|
<v t="aum.20040813212609"><vh>usage</vh></v>
|
|
<v t="aum.20040814015747"><vh>err</vh></v>
|
|
<v t="aum.20040813211551" a="TV"><vh>main</vh></v>
|
|
</v>
|
|
<v t="aum.20040803141131" a="E"><vh>MAINLINE</vh>
|
|
<v t="aum.20040812110124"><vh>mainline</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040815000938" a="E"><vh>Deployment</vh>
|
|
<v t="aum.20040814132559" a="E" tnodeList="aum.20040814132559,aum.20040814140643"><vh>@file-nosent stasher</vh>
|
|
<v t="aum.20040814140643"><vh>contents</vh></v>
|
|
</v>
|
|
<v t="aum.20040814134235" a="E" tnodeList="aum.20040814134235,aum.20040814140643"><vh>@file-nosent stasher-launch.py</vh>
|
|
<v t="aum.20040814140643"><vh>contents</vh></v>
|
|
</v>
|
|
<v t="aum.20040814133042" tnodeList="aum.20040814133042"><vh>@file release.sh</vh></v>
|
|
<v t="aum.20040814234015" tnodeList="aum.20040814234015"><vh>@file setup-stasher.py</vh></v>
|
|
<v t="aum.20040815001048" a="E" tnodeList="aum.20040815001048,aum.20040815143611,aum.20040815143611.1,aum.20040815143611.2,aum.20040815143611.3"><vh>@file-nosent README.txt</vh>
|
|
<v t="aum.20040815143611"><vh>installing-from-cvs</vh></v>
|
|
<v t="aum.20040815143611.1"><vh>doze warning</vh></v>
|
|
<v t="aum.20040815143611.2"><vh>alpha warning</vh></v>
|
|
<v t="aum.20040815143611.3"><vh>using</vh></v>
|
|
</v>
|
|
<v t="aum.20040815143009" a="E" tnodeList="aum.20040815143009,aum.20040815143611.4,aum.20040815143611.2,aum.20040815143611.3"><vh>@file-nosent README-tarball.txt</vh>
|
|
<v t="aum.20040815143611.4"><vh>installing as tarball</vh></v>
|
|
<v t="aum.20040815143611.2"><vh>alpha warning</vh></v>
|
|
<v t="aum.20040815143611.3"><vh>using</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040815000938.1" a="E"><vh>Testing</vh>
|
|
<v t="aum.20040813202242" tnodeList="aum.20040813202242,aum.20040813203116,aum.20040813203621,aum.20040807013038.9,aum.20040811013733,aum.20040812013918,aum.20040813203339,aum.20040813203339.1,aum.20040804140615,aum.20040804142640,aum.20040804140958,aum.20040812125235,aum.20040804141700,aum.20040804141700.1,aum.20040812015208,aum.20040804141801,aum.20040804141824,aum.20040809222157,aum.20040810142448,aum.20040812221935,aum.20040812230933,aum.20040812020746,aum.20040810122748,aum.20040810125759,aum.20040810132855,aum.20040810131309,aum.20040810142611,aum.20040813013718,aum.20040810115020,aum.20040810141626,aum.20040806232701,aum.20040806232714,aum.20040813202517,aum.20040803141131.1,aum.20040813211933,aum.20040811160223,aum.20040811223457,aum.20040813202541,aum.20040813202541.1"><vh>@file ktest.py</vh>
|
|
<v t="aum.20040813203116"><vh>imports</vh></v>
|
|
<v t="aum.20040813203621"><vh>constants</vh></v>
|
|
<v t="aum.20040807013038.9" a="E"><vh>TEST NETWORK</vh>
|
|
<v t="aum.20040811013733"><vh>class KTestSocket</vh></v>
|
|
<v t="aum.20040812013918"><vh>class KTestMap</vh></v>
|
|
<v t="aum.20040813203339"><vh>class KCore</vh></v>
|
|
<v t="aum.20040813203339.1"><vh>class KNode</vh></v>
|
|
<v t="aum.20040804140615" a="E"><vh>class KTestNetwork</vh>
|
|
<v t="aum.20040804142640"><vh>attribs</vh></v>
|
|
<v t="aum.20040804140958"><vh>__init__</vh></v>
|
|
<v t="aum.20040812125235"><vh>__del__</vh></v>
|
|
<v t="aum.20040804141700"><vh>__getitem__</vh></v>
|
|
<v t="aum.20040804141700.1"><vh>__len__</vh></v>
|
|
<v t="aum.20040812015208"><vh>connect</vh></v>
|
|
<v t="aum.20040804141801"><vh>start</vh></v>
|
|
<v t="aum.20040804141824"><vh>stop</vh></v>
|
|
<v t="aum.20040809222157"><vh>dump</vh></v>
|
|
<v t="aum.20040810142448"><vh>dumplong</vh></v>
|
|
<v t="aum.20040812221935"><vh>findpath</vh></v>
|
|
<v t="aum.20040812230933"><vh>testconnectivity</vh></v>
|
|
<v t="aum.20040812020746"><vh>purge</vh></v>
|
|
<v t="aum.20040810122748"><vh>whohas</vh></v>
|
|
<v t="aum.20040810125759"><vh>whocanfind</vh></v>
|
|
<v t="aum.20040810132855"><vh>findnode</vh></v>
|
|
<v t="aum.20040810131309"><vh>closestto</vh></v>
|
|
<v t="aum.20040810142611"><vh>getPeer</vh></v>
|
|
<v t="aum.20040813013718"><vh>getPeerIdx</vh></v>
|
|
<v t="aum.20040810115020"><vh>getPeerName</vh></v>
|
|
<v t="aum.20040810141626"><vh>dumpids</vh></v>
|
|
<v t="aum.20040806232701"><vh>__str__</vh></v>
|
|
<v t="aum.20040806232714"><vh>__repr__</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040813202517" a="E"><vh>Funcs</vh>
|
|
<v t="aum.20040803141131.1"><vh>test</vh></v>
|
|
<v t="aum.20040813211933"><vh>test1</vh></v>
|
|
<v t="aum.20040811160223"><vh>debug</vh></v>
|
|
<v t="aum.20040811223457"><vh>doput</vh></v>
|
|
</v>
|
|
<v t="aum.20040813202541" a="E"><vh>MAINLINE</vh>
|
|
<v t="aum.20040813202541.1"><vh>mainline</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040814191506" tnodeList="aum.20040814191506"><vh>@file node1.sh</vh></v>
|
|
<v t="aum.20040814191718" tnodeList="aum.20040814191718"><vh>@file node2.sh</vh></v>
|
|
</v>
|
|
<v t="aum.20040815000938.2" a="E"><vh>Utility</vh>
|
|
<v t="aum.20040805155412"><vh>@file pybloom.py</vh>
|
|
<v t="aum.20040805155412.1"><vh>imports</vh></v>
|
|
<v t="aum.20040805155412.2"><vh>globals</vh></v>
|
|
<v t="aum.20040805155412.3"><vh>mixarray_init</vh></v>
|
|
<v t="aum.20040805155412.4" a="E"><vh>class Bloom</vh>
|
|
<v t="aum.20040805160344"><vh>attribs</vh></v>
|
|
<v t="aum.20040805155412.5"><vh>__init__</vh></v>
|
|
<v t="aum.20040805155412.6"><vh>_make_array</vh></v>
|
|
<v t="aum.20040805155412.7"><vh>_hashfunc</vh></v>
|
|
<v t="aum.20040805155412.8"><vh>insert</vh></v>
|
|
<v t="aum.20040805155412.9"><vh>__contains__</vh></v>
|
|
</v>
|
|
<v t="aum.20040805155412.10" a="E"><vh>class CountedBloom</vh>
|
|
<v t="aum.20040805155412.11"><vh>__init__</vh></v>
|
|
<v t="aum.20040805155412.12"><vh>insert</vh></v>
|
|
<v t="aum.20040805155412.13"><vh>__contains__</vh></v>
|
|
<v t="aum.20040805155412.14"><vh>__delitem__</vh></v>
|
|
</v>
|
|
<v t="aum.20040805155412.15"><vh>mainline</vh></v>
|
|
</v>
|
|
<v t="aum.20040808213724" tnodeList="aum.20040808213724,aum.20040808221610,aum.20040808221925,aum.20040809135303,aum.20040809162443,aum.20040809135303.1,aum.20040809135303.2,aum.20040809135303.3,aum.20040809145905,aum.20040809150949,aum.20040809150949.1,aum.20040808221610.1,aum.20040808223205,aum.20040808221610.3,aum.20040808221610.4,aum.20040809160231,aum.20040808231518,aum.20040808224404,aum.20040809135512,aum.20040809145111,aum.20040809135512.1"><vh>@file-nosent hashcash.py</vh>
|
|
<v t="aum.20040808221610"><vh>imports</vh></v>
|
|
<v t="aum.20040808221925"><vh>globals</vh></v>
|
|
<v t="aum.20040809135303" a="E"><vh>class HashCash</vh>
|
|
<v t="aum.20040809162443"><vh>attributes</vh></v>
|
|
<v t="aum.20040809135303.1"><vh>__init__</vh></v>
|
|
<v t="aum.20040809135303.2"><vh>generate</vh></v>
|
|
<v t="aum.20040809135303.3"><vh>verify</vh></v>
|
|
<v t="aum.20040809145905"><vh>_checkBase64</vh></v>
|
|
<v t="aum.20040809150949"><vh>_enc64</vh></v>
|
|
<v t="aum.20040809150949.1"><vh>_dec64</vh></v>
|
|
</v>
|
|
<v t="aum.20040808221610.1"><vh>generate</vh></v>
|
|
<v t="aum.20040808223205"><vh>verify</vh></v>
|
|
<v t="aum.20040808221610.3"><vh>binify</vh></v>
|
|
<v t="aum.20040808221610.4"><vh>intify</vh></v>
|
|
<v t="aum.20040809160231"><vh>_randomString</vh></v>
|
|
<v t="aum.20040808231518"><vh>psyco</vh></v>
|
|
<v t="aum.20040808224404"><vh>test</vh></v>
|
|
<v t="aum.20040809135512"><vh>ctest</vh></v>
|
|
<v t="aum.20040809145111"><vh>ntest</vh></v>
|
|
<v t="aum.20040809135512.1"><vh>mainline</vh></v>
|
|
</v>
|
|
</v>
|
|
<v t="aum.20040804223241"><vh>JUNK</vh>
|
|
<v t="aum.20040811162256" a="E"><vh>Findnode RPC on_reply</vh>
|
|
<v t="aum.20040811162302"><vh>on_reply</vh></v>
|
|
</v>
|
|
<v t="aum.20040804204524"><vh>class KPendingResultBase</vh>
|
|
<v t="aum.20040804204524.1"><vh>__init__</vh></v>
|
|
<v t="aum.20040804210421"><vh>append</vh></v>
|
|
<v t="aum.20040804210911"><vh>wait</vh></v>
|
|
<v t="aum.20040804211058"><vh>check</vh></v>
|
|
<v t="aum.20040804212714"><vh>destroySelf</vh></v>
|
|
<v t="aum.20040804211248"><vh>on_tick</vh></v>
|
|
<v t="aum.20040804211539"><vh>on_packet</vh></v>
|
|
<v t="aum.20040804205740"><vh>__cmp__</vh></v>
|
|
</v>
|
|
<v t="aum.20040804212126"><vh>class KPendingResultPing</vh>
|
|
<v t="aum.20040804212126.1"><vh>__init__</vh></v>
|
|
<v t="aum.20040804212342"><vh>on_tick</vh></v>
|
|
<v t="aum.20040804212607"><vh>on_packet</vh></v>
|
|
</v>
|
|
</v>
|
|
</v>
|
|
</vnodes>
|
|
<tnodes>
|
|
<t tx="aum.20040802153926"></t>
|
|
<t tx="aum.20040802154007">@first #! /usr/bin/env python
|
|
"""
|
|
A simple implementation of the
|
|
U{Kademlia<http://www.infoanarchy.org/wiki/wiki.pl?Kademlia>}
|
|
P2P distributed storage and retrieval protocol, designed to
|
|
utilise the U{I2P<http://www.i2p.net>} stealth network as its transport.
|
|
|
|
Most application developers will only need to know about the L{KNode} class
|
|
"""
|
|
|
|
# I strongly recommend that when editing this file, you use the Leo
|
|
# outlining and literate programming editor - http://leo.sf.net
|
|
# If Leo doesn't agree with your religion, please try to leave the markups intact
|
|
|
|
@others
|
|
|
|
</t>
|
|
<t tx="aum.20040802154241"># define our exceptions
|
|
|
|
class KValueTooLarge(Exception):
|
|
"""
|
|
Trying to insert a value of excessive size into the network.
|
|
Maximum key size is L{maxValueSize}
|
|
"""
|
|
|
|
class KBadHash(Exception):
|
|
"""
|
|
Invalid hash string
|
|
"""
|
|
|
|
class KNotImplemented(Exception):
|
|
"""
|
|
A required method was not implemented
|
|
"""
|
|
|
|
class KBadNode(Exception):
|
|
"""
|
|
Invalid Node object
|
|
"""
|
|
|
|
class KBadPeer(Exception):
|
|
"""
|
|
Invalid Peer object - should be a KPeer
|
|
"""
|
|
|
|
class KBadDest(Exception):
|
|
"""Invalid I2P Node Dest"""
|
|
|
|
</t>
|
|
<t tx="aum.20040802154241.1">class KHash(KBase):
|
|
"""
|
|
Wraps 160-bit hashes as abstract objects, on which
|
|
operations such as xor, <, >, etc can be performed.
|
|
|
|
Kademlia node ids and keys are held as objects
|
|
of this class.
|
|
|
|
Internally, hashes are stored as python long ints
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040802154241.2">def __init__(self, val=None, **kw):
|
|
"""
|
|
Create a new hash object.
|
|
|
|
val can be one of the following:
|
|
- None (default) - a random value will be created
|
|
- long int - this will be used as the raw hash
|
|
- string - the string will be hashed and stored
|
|
- another KHash object - its value will be taken
|
|
- a KNode or KPeer object - its hash will be taken
|
|
|
|
If val is not given, a raw hash value can be passed in
|
|
with the keyword 'raw'. Such value must be a python long int
|
|
or a 20-char string
|
|
"""
|
|
self.value = 0L
|
|
if val:
|
|
if isinstance(val, KHash):
|
|
self.value = val.value
|
|
elif type(val) in [type(0), type(0L)]:
|
|
self.value = long(val)
|
|
elif isinstance(val, KNode) or isinstance(val, KPeer):
|
|
self.value = val.id.value
|
|
else:
|
|
raw = self.raw = shahash(val, bin=1)
|
|
for c in raw:
|
|
self.value = self.value * 256 + ord(c)
|
|
else:
|
|
rawval = kw.get('raw', None)
|
|
if rawval == None:
|
|
# generate random
|
|
random.seed()
|
|
for i in range(20):
|
|
self.value = self.value * 256 + random.randint(0, 256)
|
|
elif type(rawval) in [type(0), type(0L)]:
|
|
self.value = long(rawval)
|
|
elif type(rawval) == type(""):
|
|
if len(rawval) == 20:
|
|
for i in rawval:
|
|
self.value = self.value * 256 + ord(i)
|
|
elif len(rawval) == 40:
|
|
try:
|
|
self.value = long(rawval, 16)
|
|
except:
|
|
raise KBadHash(rawval)
|
|
else:
|
|
raise KBadHash(rawval)
|
|
else:
|
|
print "rawval=%s %s %s" % (type(rawval), rawval.__class__, repr(rawval))
|
|
raise KBadHash(rawval)
|
|
|
|
</t>
|
|
<t tx="aum.20040802154945">import sys, os, types, sha, random, threading, thread, traceback, Queue
|
|
import time, math, random, pickle, getopt, re
|
|
import signal
|
|
|
|
# some windows-specifics (yggghh)
|
|
if sys.platform == 'win32':
|
|
try:
|
|
import win32api
|
|
import win32process
|
|
import _winreg
|
|
except:
|
|
print "Python win32 extensions not installed."
|
|
print "Please go to http://sourceforge.net/project/showfiles.php?group_id=78018"
|
|
print "and download/install the file pywin32-202.win32-py%s.%s.exe" % \
|
|
sys.version_info[:2]
|
|
sys.exit(1)
|
|
|
|
from StringIO import StringIO
|
|
from pdb import set_trace
|
|
|
|
try:
|
|
import bencode
|
|
except:
|
|
print "The bencode module is missing from your python installation."
|
|
print "Are you sure you installed Stasher correctly?"
|
|
sys.exit(1)
|
|
|
|
try:
|
|
import i2p.socket
|
|
import i2p.select
|
|
import i2p.pylib
|
|
SocketServer = i2p.pylib.SocketServer
|
|
socket = i2p.pylib.socket
|
|
except:
|
|
print "You don't appear to have the I2P Python modules installed."
|
|
print "Not good. Stasher totally needs them."
|
|
print "Please to to i2p/apps/sam/python in your I2P cvs tree, and"
|
|
print "install the core I2P python modules first"
|
|
sys.exit(1)
|
|
|
|
</t>
|
|
<t tx="aum.20040802155814.2">class KPeer(KBase):
|
|
"""
|
|
Encapsulates a peer node of a L{KNode},
|
|
storing its ID and contact info
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040802155814.3">def __init__(self, node, dest):
|
|
"""
|
|
Create a ref to a kademlia peer node
|
|
|
|
Arguments:
|
|
- node - reference to node which has the relationship
|
|
to this peer
|
|
- dest - the peer's I2P destination, as base64
|
|
"""
|
|
if not isinstance(node, KNode):
|
|
raise KBadNode(node)
|
|
if not isinstance(dest, str):
|
|
raise KBadDest(dest)
|
|
|
|
self.node = node
|
|
self.dest = dest
|
|
self.id = KHash(dest)
|
|
|
|
self.justSeen()
|
|
|
|
</t>
|
|
<t tx="aum.20040802160335">def send_ping(self, **kw):
|
|
"""
|
|
Sends a ping to remote peer
|
|
"""
|
|
self.send_raw(type="ping", **kw)
|
|
</t>
|
|
<t tx="aum.20040802160828">def send_store(self, **kw):
|
|
"""
|
|
sends a store command to peer
|
|
"""
|
|
self.log(4, "\npeer %s\ndest %s...\nsending store cmd: %s" % (self, self.dest[:12], repr(kw)))
|
|
|
|
self.send_raw(type="store", **kw)
|
|
|
|
</t>
|
|
<t tx="aum.20040802160828.1">def send_findNode(self, hash, **kw):
|
|
"""
|
|
sends a findNode command to peer
|
|
"""
|
|
if not isinstance(hash, KHash):
|
|
raise KBadHash
|
|
|
|
self.log(5, "\nquerying peer %s\ntarget hash %s" % (self, hash))
|
|
|
|
self.send_raw(type="findNode", hash=hash.value, **kw)
|
|
|
|
</t>
|
|
<t tx="aum.20040802160828.2">def send_findData(self, hash, **kw):
|
|
"""
|
|
sends a findData command to peer
|
|
"""
|
|
if not isinstance(hash, KHash):
|
|
raise KBadHash
|
|
|
|
self.log(5, "\nquerying peer %s\ntarget hash %s" % (self, hash))
|
|
|
|
self.send_raw(type="findData", hash=hash.value, **kw)
|
|
|
|
</t>
|
|
<t tx="aum.20040802161601">class KNode(KBase):
|
|
"""
|
|
B{Public API to this Kademlia implementation}
|
|
|
|
You should not normally need to use, or even be aware of,
|
|
any of the other classes
|
|
|
|
And in this class, the only methods you need to worry about are:
|
|
- L{start} - starts the node running
|
|
- L{stop} - stops the node
|
|
- L{get} - retrieve a key value
|
|
- L{put} - stores a key value
|
|
- L{addref} - imports a noderef
|
|
|
|
This class implements a single kademlia node.
|
|
Within a single process, you can create as many nodes as you like.
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040802161601.1">def start(self, doPings=True):
|
|
"""
|
|
Starts the node running
|
|
"""
|
|
# barf if already running
|
|
if self.isRunning:
|
|
self.log(3, "node %s is already running!" % self.name)
|
|
return
|
|
|
|
self.log(3, "starting node %s" % self.name)
|
|
|
|
# first step - ping all our peers
|
|
if doPings:
|
|
for peer in self.peers:
|
|
self.log(3, "doing initial ping\n%s\n%s" % (self, peer))
|
|
KRpcPing(self, peer=peer)
|
|
|
|
# first step - do a findNode against our own node id, and ping our
|
|
# neighbours
|
|
if greetPeersOnStartup:
|
|
neighbours = KRpcFindNode(self, hash=self.id).execute()
|
|
self.log(3, "neighbours=%s" % repr([n[:10] for n in neighbours]))
|
|
for n in neighbours:
|
|
n = self._normalisePeer(n)
|
|
KRpcPing(self, peer=n)
|
|
|
|
# note now that we're running
|
|
self.isRunning = True
|
|
|
|
# and enlist with the core
|
|
if runCore:
|
|
core.subscribe(self)
|
|
else:
|
|
# central core disabled, run our own receiver thread instead
|
|
thread.start_new_thread(self._threadRx, ())
|
|
</t>
|
|
<t tx="aum.20040802161601.2">def stop(self):
|
|
"""
|
|
Shuts down the node
|
|
"""
|
|
self.isRunning = 0
|
|
if runCore:
|
|
try:
|
|
core.unsubscribe(self)
|
|
except:
|
|
pass
|
|
</t>
|
|
<t tx="aum.20040802161601.3">def get(self, item, callback=None, **kw):
|
|
"""
|
|
Attempts to retrieve data from the network
|
|
|
|
Arguments:
|
|
- item - the key we desire
|
|
- callback - optional - if given, the get will be performed
|
|
asynchronously, and callback will be invoked upon completion, with
|
|
the result as first argument
|
|
Keywords:
|
|
- local - optional - if True, limits this search to this local node
|
|
default is False
|
|
|
|
Returns:
|
|
- if no callback - the item value if the item was found, or None if not
|
|
- if callback, None is returned
|
|
"""
|
|
def processResult(r):
|
|
if isinstance(r, str):
|
|
return r
|
|
return None
|
|
|
|
if callback:
|
|
# create a func to process callback result
|
|
def onCallback(res):
|
|
callback(processResult(res))
|
|
|
|
self._finddata(item, onCallback, **kw)
|
|
else:
|
|
return processResult(self._finddata(item, **kw))
|
|
|
|
</t>
|
|
<t tx="aum.20040802161601.4">def put(self, key, value, callback=None, **kw):
|
|
"""
|
|
Inserts a named key into the network
|
|
|
|
Arguments:
|
|
- key - one of:
|
|
- None - a secure key will be generated and used
|
|
- a KHash object
|
|
- a raw string which will be hashed into a KHash object
|
|
- val - a string, the value associated with the key
|
|
|
|
Keywords:
|
|
- local - default False - if True, limits the insert to the
|
|
local node
|
|
|
|
If the value is larger than L{maxValueSize}, a L{KValueTooLarge}
|
|
exception will occur.
|
|
"""
|
|
return self._store(key, value, callback, **kw)
|
|
|
|
</t>
|
|
<t tx="aum.20040802161601.5">def __getitem__(self, item):
|
|
"""
|
|
Allows dict-like accesses on the node object
|
|
"""
|
|
return self.get(item)
|
|
</t>
|
|
<t tx="aum.20040802161601.6">def __setitem__(self, item, val):
|
|
"""
|
|
Allows dict-like key setting on the node object
|
|
"""
|
|
self.put(item, val)
|
|
|
|
</t>
|
|
<t tx="aum.20040802164545">def __str__(self):
|
|
return "<KHash: 0x%x>" % self.value
|
|
|
|
def __repr__(self):
|
|
return str(self)
|
|
|
|
</t>
|
|
<t tx="aum.20040802165054">def __eq__(self, other):
|
|
#log(2, "KHash: comparing %s to %s" % (self, other))
|
|
res = self.value == getattr(other, 'value', None)
|
|
#self.log(2, "KHash: res = %s" % repr(res))
|
|
return res
|
|
|
|
def __ne__(self, other):
|
|
return not (self == other)
|
|
|
|
def __lt__(self, other):
|
|
return self.value < other.value
|
|
|
|
def __gt__(self, other):
|
|
return self.value > other.value
|
|
|
|
def __le__(self, other):
|
|
return self.value <= other.value
|
|
|
|
def __ge__(self, other):
|
|
return self.value >= other.value
|
|
|
|
def __ne__(self, other):
|
|
return self.value != other.value
|
|
|
|
def __xor__(self, other):
|
|
return self.value ^ other.value
|
|
|
|
</t>
|
|
<t tx="aum.20040802184641">def send_raw(self, **kw):
|
|
"""
|
|
Sends a raw datagram to peer
|
|
|
|
No arguments - just keywords, all of which must be strings or
|
|
other objects which can be bencoded
|
|
"""
|
|
self.node._sendRaw(self, **kw)
|
|
</t>
|
|
<t tx="aum.20040802184641.1">def __init__(self, name, **kw):
|
|
"""
|
|
Creates a kademlia node of name 'name'.
|
|
|
|
Name is mandatory, because each name is permanently written
|
|
to the SAM bridge's store
|
|
|
|
I thought of supporting random name generation, but went off this
|
|
idea because names get permanently stored to SAM bridge's file
|
|
|
|
Arguments:
|
|
- name - mandatory - a short text name for the node, should
|
|
be alphanumerics, '-', '.', '_'
|
|
This name is used for the SAM socket session.
|
|
|
|
Keywords:
|
|
- storage - optional - an instance of L{KStorageBase} or one of
|
|
its subclasses. If not given, default action is to instantiate
|
|
a L{KStorageFile} object against the given node name
|
|
"""
|
|
# remember who we are
|
|
self.name = name
|
|
|
|
# not running yet, will launch when explicitly started, or implicitly
|
|
# when the first operation gets done
|
|
self.isRunning = False
|
|
|
|
# create socket and get its dest, and determine our node id
|
|
self.id = KHash("<NONE>")
|
|
self.log(5, "creating socket for node %s" % name)
|
|
self.log(5, "socket for node %s created" % name)
|
|
if self.SocketFactory == None:
|
|
self.SocketFactory = i2p.socket.socket
|
|
self.sock = self.SocketFactory(
|
|
"stashernode-"+name,
|
|
i2p.socket.SOCK_DGRAM,
|
|
samaddr=samAddr,
|
|
**kw)
|
|
#self.sockLock = threading.Lock() # prevents socket API reentrance
|
|
self.sock.setblocking(0)
|
|
self.dest = self.sock.dest
|
|
self.id = KHash(self.dest)
|
|
|
|
# create our buckets
|
|
self.buckets = []
|
|
for i in range(160):
|
|
self.buckets.append(KBucket())
|
|
|
|
# create our storage object, default to new instance of KStorageFile
|
|
self.storage = kw.get('storage', KStorageFile(self))
|
|
|
|
# dig out all previously known nodes
|
|
self.peers = self.storage.getRefs()
|
|
|
|
# set up dict of callers awaiting replies
|
|
# keys are (peerobj, msgId) tuples, values are Queue.Queue objects
|
|
self.pendingPings = {}
|
|
|
|
# mapping of (peer, msgId) to RPC object, so when RPC replies come in,
|
|
# they can be passed directly to the RPC object concerned
|
|
self.rpcBindings = {}
|
|
|
|
# KRpc objects waiting for peer replies - used for checking for timeouts
|
|
self.rpcPending = []
|
|
|
|
# miscellaneous shit
|
|
self._msgIdNext = 0
|
|
#self._msgIdLock = threading.Lock()
|
|
|
|
# register in global map
|
|
_nodes[name] = self
|
|
|
|
|
|
</t>
|
|
<t tx="aum.20040803005037">class KStorageBase(KBase):
|
|
"""
|
|
Base class for node storage objects
|
|
|
|
This needs to be overridden by implementation-specific
|
|
solutions.
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040803005037.1">def __init__(self, node, *args, **kw):
|
|
"""
|
|
Override this method
|
|
|
|
First argument should be a node instance
|
|
"""
|
|
raise KNotImplemented
|
|
|
|
</t>
|
|
<t tx="aum.20040803005037.2">def putRefs(self, *refs):
|
|
"""
|
|
Saves one or more noderefs
|
|
|
|
Arguments:
|
|
- zero or more KPeer objects, or lists or tuples of objects
|
|
"""
|
|
raise KNotImplemented
|
|
</t>
|
|
<t tx="aum.20040803005037.3">def getRefs(self):
|
|
"""
|
|
Returns a list of KPeer objects, comprising refs
|
|
of peers known to this node
|
|
"""
|
|
raise KNotImplemented
|
|
|
|
</t>
|
|
<t tx="aum.20040803005037.4">def putKey(self, key, value):
|
|
"""
|
|
Stores value, a string, into the local storage
|
|
under key 'key'
|
|
"""
|
|
raise KNotImplemented
|
|
|
|
</t>
|
|
<t tx="aum.20040803005309">def getKey(self, key):
|
|
"""
|
|
Attempts to retrieve item from node's local, which was
|
|
stored with key 'key'.
|
|
|
|
Returns value as a string if found, or None if not present
|
|
"""
|
|
raise KNotImplemented
|
|
</t>
|
|
<t tx="aum.20040803010321">class KStorageFile(KStorageBase):
|
|
"""
|
|
Implements node-local storage, using the local filesystem,
|
|
with the following hierarchy:
|
|
|
|
- HOME ( ~ in linux, some other shit for windows)
|
|
- .i2pkademlia
|
|
- <nodename>
|
|
- noderefs
|
|
- <node1 base64 hash>
|
|
- contains node dest, and other shit
|
|
- ...
|
|
- keys
|
|
- <keyname1>
|
|
- contains raw key value
|
|
- ...
|
|
|
|
This is one ugly sukka, perhaps a db4, mysql etc implementation
|
|
would be better.
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040803010321.1">def __init__(self, node, storeDir=None):
|
|
"""
|
|
Creates a persistent storage object for node
|
|
'nodeName', based at directory 'storeDir' (default
|
|
is nodeDir
|
|
"""
|
|
self.node = node
|
|
self.nodeName = node.name
|
|
|
|
if storeDir == None:
|
|
# work out local directory
|
|
self.topDir = userI2PDir()
|
|
|
|
# add node dir and subdirs
|
|
self.nodeDir = userI2PDir(self.nodeName)
|
|
|
|
self.refsDir = os.path.join(self.nodeDir, "noderefs")
|
|
if not os.path.isdir(self.refsDir):
|
|
os.makedirs(self.refsDir)
|
|
|
|
self.keysDir = os.path.join(self.nodeDir, "keys")
|
|
if not os.path.isdir(self.keysDir):
|
|
os.makedirs(self.keysDir)
|
|
|
|
</t>
|
|
<t tx="aum.20040803013434">class KBucket(KBase):
|
|
"""
|
|
Implements the 'k-bucket' object as required in Kademlia spec
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040803013434.1">def __init__(self):
|
|
"""
|
|
Creates a single k-bucket
|
|
"""
|
|
# list of known nodes
|
|
# order is least recently seen at head, most recently seen at tail
|
|
self.nodes = []
|
|
|
|
# list of death-row records
|
|
# refer spec section 2.1, paragraph 2
|
|
# we deviate a little:
|
|
# when we hear from a new peer, and the bucket is full,
|
|
# we temporarily displace the old peer, and stick the new
|
|
# peer at end of list, then send out a ping
|
|
# If we hear from the old peer within a reasonable time,
|
|
# the new peer gets evicted and replaced with the old peer
|
|
#
|
|
# this list holds 2-tuples (oldpeer, newpeer), where
|
|
# oldpeer is the least-recently-seen peer that we displaced, and
|
|
# newpeer is the new peer we just heard from.
|
|
self.deathrow = []
|
|
|
|
</t>
|
|
<t tx="aum.20040803013638">
|
|
# --------------------------------------------
|
|
# START USER-CONFIGURABLE CONSTANTS
|
|
# --------------------------------------------
|
|
|
|
# host:port to connect to I2P SAM Bridge
|
|
samAddr = i2p.socket.samaddr
|
|
|
|
# host:port to listen on for command line client
|
|
clientAddr = "127.0.0.1:7659"
|
|
|
|
defaultNodename = "0" # will be prefixed by 'stashernode'
|
|
|
|
# maximum size of each stored item
|
|
maxValueSize = 30000
|
|
|
|
# maximum number of noderefs that can be stored in a bucket
|
|
# (refer spec section 2.1, first paragraph)
|
|
maxBucketSize = 20
|
|
|
|
# number of peers to return from a search
|
|
numSearchPeers = 3
|
|
|
|
# maximum number of concurrent queries per findnode/finddata rpc
|
|
maxConcurrentQueries = 10
|
|
|
|
# number of peers to store onto
|
|
numStorePeers = 10
|
|
|
|
# Logger settings
|
|
logFile = None
|
|
logVerbosity = 2
|
|
|
|
# data directory location - set to a path to override the default
|
|
# which is the user's home dir
|
|
dataDir = None
|
|
|
|
# whether a node, on startup, should do a findnode on itself to
|
|
# locate its closest neighbours
|
|
greetPeersOnStartup = False
|
|
#greetPeersOnStartup = True
|
|
|
|
# multi-purpose testing flag
|
|
testing = False
|
|
#testing = True
|
|
|
|
tunnelDepth = 0
|
|
|
|
# set to True to enable single handler thread that manages all nodes,
|
|
# or False to make each node run its own handler thread
|
|
#runCore = False
|
|
runCore = True
|
|
|
|
# timeouts - calibrate as needed
|
|
timeout = {
|
|
'ping' : 120,
|
|
'findNode' : 120,
|
|
'findData' : 120,
|
|
'store' : 120,
|
|
}
|
|
|
|
logToSocket = None
|
|
|
|
desperatelyDebugging = False
|
|
|
|
if desperatelyDebugging:
|
|
runCoreInBackground = False
|
|
else:
|
|
runCoreInBackground = True
|
|
|
|
# --------------------------------------------
|
|
# END OF USER-CONFIGURABLE CONSTANTS
|
|
# --------------------------------------------
|
|
|
|
# ----------------------------------------------
|
|
# hack anything below this line at your own risk
|
|
|
|
</t>
|
|
<t tx="aum.20040803013638.1">def justSeenPeer(self, peer):
|
|
"""
|
|
Tells the bucket that we've just seen a given node
|
|
"""
|
|
nodes = self.nodes
|
|
|
|
if not isinstance(peer, KPeer):
|
|
raise KBadNode
|
|
|
|
try:
|
|
idx = nodes.index(peer)
|
|
except:
|
|
idx = -1
|
|
if idx >= 0:
|
|
del nodes[idx]
|
|
nodes.append(peer)
|
|
else:
|
|
nodes.append(peer)
|
|
|
|
# might at some time need to implement death-row logic
|
|
# when we set a bucket size limit - refer __init__
|
|
</t>
|
|
<t tx="aum.20040803015148">def addref(self, peer, doPing=False):
|
|
"""
|
|
Given a peer node's destination, add it to our
|
|
buckets and internal data store
|
|
|
|
Arguments:
|
|
- peer - one of:
|
|
- the I2P destination of the peer node, as
|
|
a base64 string
|
|
- a KNode object
|
|
- a KPeer object
|
|
- doPing - ping this node automatically (default False)
|
|
"""
|
|
peer = self._normalisePeer(peer)
|
|
|
|
# remember peer if not already known
|
|
if peer.dest == self.dest:
|
|
self.log(3, "node %s, trying to add ref to ourself???" % self.name)
|
|
return peer
|
|
elif not self._findPeer(peer.dest):
|
|
self.peers.append(peer)
|
|
self.storage.putRefs(peer)
|
|
else:
|
|
self.log(4, "node %s, trying to add duplicate noderef %s" % (
|
|
self.name, peer))
|
|
return peer
|
|
|
|
# update our KBucket
|
|
dist = self.id.distance(peer.id)
|
|
self.buckets[dist].justSeenPeer(peer)
|
|
|
|
if doPing:
|
|
self.log(4, "doing initial ping\n%s\n%s" % (self, peer))
|
|
KRpcPing(self, peer=peer)
|
|
|
|
return peer
|
|
|
|
</t>
|
|
<t tx="aum.20040803131111.2"></t>
|
|
<t tx="aum.20040803131210"></t>
|
|
<t tx="aum.20040803131210.1">def _sendRaw(self, peer, **kw):
|
|
"""
|
|
Serialises keywords passed, and sends this as a datagram
|
|
to node 'peer'
|
|
"""
|
|
# update our KBucket
|
|
dist = self.id.distance(peer.id)
|
|
self.buckets[dist].justSeenPeer(peer)
|
|
|
|
# save ref to this peer
|
|
self.addref(peer)
|
|
|
|
params = dict(kw)
|
|
msgId = params.get('msgId', None)
|
|
if msgId == None:
|
|
msgId = params['msgId'] = self._msgIdAlloc()
|
|
|
|
objenc = messageEncode(params)
|
|
self.log(5, "node %s waiting for send lock" % self.name)
|
|
#self.sockLock.acquire()
|
|
self.log(5, "node %s got send lock" % self.name)
|
|
try:
|
|
self.sock.sendto(objenc, 0, peer.dest)
|
|
except:
|
|
traceback.print_exc()
|
|
#self.sockLock.release()
|
|
self.log(5, "node %s released send lock" % self.name)
|
|
|
|
self.log(4, "node %s sent %s to peer %s" % (self.name, params, peer.dest))
|
|
return msgId
|
|
|
|
</t>
|
|
<t tx="aum.20040803131210.2"></t>
|
|
<t tx="aum.20040803131210.3">def _threadRx(self):
|
|
"""
|
|
Thread which listens for incoming datagrams and actions
|
|
accordingly
|
|
"""
|
|
self.log(3, "starting receiver thread for node %s" % self.name)
|
|
|
|
try:
|
|
# loop to drive the node
|
|
while self.isRunning:
|
|
self._doChug()
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(3, "node %s - THREAD CRASH!" % self.name)
|
|
|
|
self.log(3, "receiver thread for node %s terminated" % self.name)
|
|
|
|
</t>
|
|
<t tx="aum.20040803132156">@others
|
|
</t>
|
|
<t tx="aum.20040803132528"># keep a dict of existing nodes, so we can prevent
|
|
# client progs from creating 2 nodes of the same name
|
|
_nodes = {}
|
|
|
|
version = "0.0.1"
|
|
|
|
</t>
|
|
<t tx="aum.20040803132528.1">logLock = threading.Lock()
|
|
|
|
def log(verbosity, msg, nPrev=0, clsname=None):
|
|
|
|
global logToSocket, logFile
|
|
|
|
# create logfile if not exists
|
|
if logFile == None:
|
|
logFile = os.path.join(userI2PDir(), "stasher.log")
|
|
|
|
# rip the stack
|
|
caller = traceback.extract_stack()[-(2+nPrev)]
|
|
path, line, func = caller[:3]
|
|
path = os.path.split(path)[1]
|
|
|
|
#print "func is type %s, val %s" % (type(func), repr(func))
|
|
|
|
#if hasattr(func, "im_class"):
|
|
# func =
|
|
|
|
if clsname:
|
|
func = clsname + "." + func
|
|
|
|
#msg = "%s:%s:%s(): %s" % (
|
|
# path,
|
|
# line,
|
|
# func,
|
|
# msg.replace("\n", "\n + "))
|
|
|
|
msg = "%s():%s: %s" % (
|
|
func,
|
|
line,
|
|
msg.replace("\n", "\n + "))
|
|
|
|
# do better logging later
|
|
if verbosity > logVerbosity:
|
|
return
|
|
|
|
if logToSocket:
|
|
try:
|
|
if isinstance(logToSocket, int):
|
|
portnum = logToSocket
|
|
logToSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
connected = 0
|
|
while 1:
|
|
try:
|
|
logToSocket.connect(("localhost", portnum))
|
|
break
|
|
except socket.error:
|
|
print "Please open an xterm/nc listening on %s" % logToSocket
|
|
time.sleep(1)
|
|
|
|
logToSocket.send(msg+"\n")
|
|
except:
|
|
traceback.print_exc()
|
|
else:
|
|
print msg
|
|
|
|
logLock.acquire()
|
|
file(logFile, "a+").write(msg + "\n")
|
|
logLock.release()
|
|
</t>
|
|
<t tx="aum.20040803134102">def _findPeer(self, dest):
|
|
"""
|
|
Look up our table of current peers for a given dest.
|
|
|
|
If dest is found, return its object, otherwise return None
|
|
"""
|
|
for peerObj in self.peers:
|
|
if peerObj.dest == dest:
|
|
return peerObj
|
|
return None
|
|
|
|
</t>
|
|
<t tx="aum.20040803134418">def putRefs(self, *args):
|
|
"""
|
|
Saves one or more noderefs into filesystem
|
|
|
|
Arguments:
|
|
- zero or more KPeer objects, or lists or tuples of objects
|
|
"""
|
|
lst = self._expandRefsList(args)
|
|
for item in lst:
|
|
b64hash = shahash(item.dest)
|
|
itemPath = os.path.join(self.refsDir, b64hash)
|
|
itemDict = {'dest':item.dest} # might need to expand later
|
|
itemPickle = bencode.bencode(itemDict)
|
|
file(itemPath, "wb").write(itemPickle)
|
|
pass
|
|
</t>
|
|
<t tx="aum.20040803134838">@others
|
|
</t>
|
|
<t tx="aum.20040803134838.1">def _expandRefsList(self, args, lst=None):
|
|
"""
|
|
Takes a sequence of args, each of which can be a KPeer
|
|
object, or a list or tuple of KPeer objects, and expands
|
|
this into a flat list
|
|
"""
|
|
if lst == None:
|
|
lst = []
|
|
for item in args:
|
|
if type(item) in [type(()), type([])]:
|
|
self._expandRefsList(item, lst)
|
|
else:
|
|
lst.append(item)
|
|
return lst
|
|
|
|
</t>
|
|
<t tx="aum.20040803140601">def getRefs(self):
|
|
"""
|
|
Returns a list of KPeer objects, comprising refs
|
|
of peers known to this node
|
|
|
|
These are read from the directory self.refsDir.
|
|
Any that can't be unpickled and instantiated are dropped, but logged
|
|
"""
|
|
peers = []
|
|
for f in os.listdir(self.refsDir):
|
|
|
|
path = os.path.join(self.refsDir, f)
|
|
pickled = file(path, "rb").read()
|
|
try:
|
|
d = bencode.bdecode(pickled)
|
|
except:
|
|
self.log(3, "node %s, bad pickle ref file %s" % (
|
|
self.nodeName, f))
|
|
continue
|
|
|
|
# instantiate a peer object
|
|
try:
|
|
peer = KPeer(self.node, d['dest'])
|
|
except:
|
|
self.log(3, "node %s, bad unpickled ref file %s" % (
|
|
self.nodeName, f))
|
|
continue
|
|
|
|
# success
|
|
peers.append(peer)
|
|
|
|
return peers
|
|
|
|
</t>
|
|
<t tx="aum.20040803141131">@others
|
|
</t>
|
|
<t tx="aum.20040803141131.1">def test(numnodes=10):
|
|
|
|
global n, n0, res
|
|
|
|
stasher.logVerbosity = 3
|
|
stasher.core = KCore(1)
|
|
|
|
os.system("rm -rf ~/.i2pkademlia")
|
|
|
|
n = KTestNetwork(10, purge=1)
|
|
n.connect()
|
|
n.start()
|
|
n0 = n[0]
|
|
|
|
return
|
|
|
|
if 0:
|
|
core.fg = True
|
|
n = KTestNetwork(10, purge=1)
|
|
n.connect()
|
|
n.start()
|
|
core.cycle()
|
|
|
|
print "about to insert"
|
|
set_trace()
|
|
n[0].put('roses', 'red')
|
|
print "insert completed"
|
|
#set_trace()
|
|
|
|
return
|
|
|
|
#set_trace()
|
|
#n[0].put('roses', 'red')
|
|
|
|
#print n[0].get('roses')
|
|
|
|
if 0:
|
|
successes = []
|
|
failures = []
|
|
for idx in range(numnodes):
|
|
|
|
# build test network of known topology
|
|
print "Building test network"
|
|
n = KTestNetwork(numnodes, purge=1)
|
|
n.connect()
|
|
n.start()
|
|
core.n = n
|
|
n.trigger = 0
|
|
|
|
if 0:
|
|
print n[0]._findnode('roses')
|
|
break
|
|
|
|
if 1:
|
|
# store something
|
|
print "storing something"
|
|
|
|
n[0].put('roses', 'red')
|
|
|
|
# try to retrieve it from node i
|
|
print "trying to retrieve it from node %s" % idx
|
|
if n[idx].get('roses') != None:
|
|
print "Node %s retrieved ok" % idx
|
|
successes.append(idx)
|
|
else:
|
|
print "Node %s failed to retrieve" % idx
|
|
failures.append(idx)
|
|
|
|
del n
|
|
|
|
print "Successful nodes: %s" % " ".join([str(x) for x in successes])
|
|
print "Failed nodes: %s" % " ".join([str(x) for x in failures])
|
|
|
|
if 0 and desperatelyDebugging:
|
|
|
|
while not n.trigger:
|
|
time.sleep(1)
|
|
|
|
n.trigger = 0
|
|
n[0].put('roses', 'red')
|
|
|
|
while not n.trigger:
|
|
time.sleep(1)
|
|
|
|
print "retrieving 'roses'"
|
|
print n[0].get('roses')
|
|
|
|
</t>
|
|
<t tx="aum.20040803142127">@others
|
|
</t>
|
|
<t tx="aum.20040803142127.1">def __str__(self):
|
|
return "<KNode:%s=0x%s...>" % (
|
|
self.name,
|
|
("%x" % self.id.value)[:8],
|
|
)
|
|
</t>
|
|
<t tx="aum.20040803142127.2">def __repr__(self):
|
|
return str(self)
|
|
|
|
</t>
|
|
<t tx="aum.20040803143007">@others
|
|
</t>
|
|
<t tx="aum.20040803143007.1">def __str__(self):
|
|
|
|
return "<KPeer:%s=>0x%s... dest %s...>" % (
|
|
self.node.name, ("%x" % self.id.value)[:8], self.dest[:8])
|
|
|
|
</t>
|
|
<t tx="aum.20040803143007.2">def __repr__(self):
|
|
|
|
return str(self)
|
|
|
|
</t>
|
|
<t tx="aum.20040803144052">@others
|
|
</t>
|
|
<t tx="aum.20040803163704">def _on_ping(self, peer, msgId, **kw):
|
|
"""
|
|
Handler for ping received events
|
|
"""
|
|
KRpcPing(self, (peer, msgId), local=True, **kw)
|
|
return
|
|
|
|
# old stuff
|
|
|
|
self.log(3, "\nnode %s\nfrom %s\nreceived:\n%s" % (self.name, peer, kw))
|
|
|
|
# randomly cause ping timeouts if testing
|
|
if testing:
|
|
howlong = random.randint(0, 5)
|
|
self.log(3, "deliberately pausing for %s seconds" % howlong)
|
|
time.sleep(howlong)
|
|
|
|
# pong back to node
|
|
peer.send_reply(msgId=msgId)
|
|
|
|
</t>
|
|
<t tx="aum.20040803163704.1">def _on_unknown(self, peer, msgId, **kw):
|
|
"""
|
|
Handler for unknown events
|
|
"""
|
|
self.log(3, "node %s from %s received msgId=%s:\n%s" % (
|
|
self.name, peer, msgId, kw))
|
|
|
|
</t>
|
|
<t tx="aum.20040803193605">def _msgIdAlloc(self):
|
|
"""
|
|
issue a new and unique message id
|
|
"""
|
|
#self._msgIdLock.acquire()
|
|
msgId = self._msgIdNext
|
|
self._msgIdNext += 1
|
|
#self._msgIdLock.release()
|
|
return msgId
|
|
</t>
|
|
<t tx="aum.20040803200819">def _normalisePeer(self, peer):
|
|
"""
|
|
Takes either a b64 dest string, a KPeer object or a KNode object,
|
|
and returns a KPeer object
|
|
"""
|
|
# act according to whatever type we're given
|
|
if isinstance(peer, KPeer):
|
|
return peer # already desired format
|
|
elif isinstance(peer, KNode):
|
|
return KPeer(self, peer.dest)
|
|
elif isinstance(peer, str) and len(peer) > 256:
|
|
return KPeer(self, peer)
|
|
else:
|
|
self.log(3, "node %s, trying to add invalid noderef %s" % (
|
|
self.name, peer))
|
|
raise KBadNode(peer)
|
|
|
|
</t>
|
|
<t tx="aum.20040803201812.1">def _ping(self, peer=None, callback=None, **kw):
|
|
"""
|
|
Sends a ping to remote peer, and awaits response
|
|
|
|
Not of much real use to application level, except
|
|
perhaps for testing
|
|
|
|
If the argument 'peer' is not given, the effect is to 'ping the
|
|
local node', which I guess might be a bit silly
|
|
|
|
The second argument 'callback' is a callable, which if given, makes this
|
|
an asynchronous (non-blocking) call, in which case the callback will be
|
|
invoked upon completion (or timeout).
|
|
|
|
If the keyword 'cbArgs' is given in addition to the callback, the callback
|
|
will fire with the results as first argument and this value as second arg
|
|
"""
|
|
if callback:
|
|
KRpcPing(self, callback, peer=peer, **kw)
|
|
else:
|
|
return KRpcPing(self, peer=peer).execute()
|
|
|
|
</t>
|
|
<t tx="aum.20040804001454">def distance(self, other):
|
|
"""
|
|
calculates the 'distance' between this hash and another hash,
|
|
and returns it as i (where distance = 2^i, and 0 <= i < 160)
|
|
"""
|
|
|
|
#log(4, "comparing: %s\nwith %s" % (self.value, other.value))
|
|
|
|
rawdistance = self.value ^ other.value
|
|
if not rawdistance:
|
|
return 0
|
|
|
|
return int(math.log(rawdistance, 2))
|
|
|
|
</t>
|
|
<t tx="aum.20040804013647">def _on_findNode(self, peer, msgId, **kw):
|
|
"""
|
|
Handles incoming findNode command
|
|
"""
|
|
KRpcFindNode(self, (peer, msgId), local=True, **kw)
|
|
|
|
</t>
|
|
<t tx="aum.20040804014115.1">def _on_findData(self, peer, msgId, **kw):
|
|
"""
|
|
Handles incoming findData command
|
|
"""
|
|
KRpcFindData(self, (peer, msgId), local=True, **kw)
|
|
|
|
</t>
|
|
<t tx="aum.20040804014643">def justSeen(self):
|
|
self.timeLastSeen = time.time()
|
|
|
|
</t>
|
|
<t tx="aum.20040804122542">class KCore(KBase):
|
|
"""
|
|
Singleton class which performs all the needed background processing.
|
|
|
|
By scheduling all processing through this object, we eliminate the
|
|
need to create threads on a per-node basis, and also make this thing
|
|
far easier to debug.
|
|
|
|
The core launches only two background threads:
|
|
- L{threadRxPackets} - listen for incoming packets bound for
|
|
any node running within a single process
|
|
- L{threadHousekeeping} - periodically invoke maintenance methods
|
|
of each node, so the node can check for timeout conditions and
|
|
other untoward happenings
|
|
|
|
These threads start up when the first node in this process is created,
|
|
and stop when the last node ceases to exist.
|
|
|
|
Upon first import, the L{stasher} module creates one instance of this
|
|
class. Upon creation, L{KNode} objects register themselves with this core.
|
|
"""
|
|
@others
|
|
|
|
</t>
|
|
<t tx="aum.20040804122542.1">def __init__(self, bg=True):
|
|
"""
|
|
Creates the I2P Kademlia core object
|
|
"""
|
|
self.bg = bg
|
|
self.fg = False
|
|
|
|
# subscribed nodes
|
|
self.nodes = []
|
|
#self.nodesLock = threading.Lock()
|
|
|
|
self.isRunning = False
|
|
self.isRunning_rx = False
|
|
|
|
</t>
|
|
<t tx="aum.20040804123117">def subscribe(self, node):
|
|
"""
|
|
Called by a node to 'subscribe' for background processing
|
|
If this is the first node, starts the handler thread
|
|
"""
|
|
#self.nodesLock.acquire()
|
|
try:
|
|
nodes = self.nodes
|
|
|
|
if node in nodes:
|
|
self.log(2, "duhhh! node already subscribed" % repr(node))
|
|
return
|
|
|
|
nodes.append(node)
|
|
|
|
if not self.isRunning:
|
|
self.isRunning = True
|
|
if self.bg and not self.fg:
|
|
self.log(3, "First node subscribing, launching threads")
|
|
thread.start_new_thread(self.threadRxPackets, ())
|
|
thread.start_new_thread(self.threadHousekeeping, ())
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(2, "exception")
|
|
|
|
#self.nodesLock.release()
|
|
|
|
</t>
|
|
<t tx="aum.20040804123350">def unsubscribe(self, node):
|
|
"""
|
|
Unsubscribes a node from the core
|
|
|
|
If this was the last node, stops the handler thread
|
|
"""
|
|
#self.nodesLock.acquire()
|
|
try:
|
|
nodes = self.nodes
|
|
|
|
if node not in nodes:
|
|
self.log(4, "duhhh! node %s was not subscribed" % repr(node))
|
|
return
|
|
|
|
self.log(2, "trying to unsubscribe node %s" % node.name)
|
|
nodes.remove(node)
|
|
|
|
if len(nodes) == 0:
|
|
self.isRunning = False
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(2, "exception")
|
|
|
|
#self.nodesLock.release()
|
|
|
|
</t>
|
|
<t tx="aum.20040804123526">def threadRxPackets(self):
|
|
"""
|
|
Sits on a select() loop, processing incoming datagrams
|
|
and actioning them appropriately.
|
|
"""
|
|
self.isRunning_rx = True
|
|
self.log(3, "KCore packet receiver thread running")
|
|
try:
|
|
while self.isRunning:
|
|
socks = [node.sock for node in self.nodes]
|
|
if desperatelyDebugging:
|
|
set_trace()
|
|
try:
|
|
inlist, outlist, errlist = self.select(socks, [], [], 1)
|
|
except KeyboardInterrupt:
|
|
self.isRunning = 0
|
|
return
|
|
|
|
self.log(5, "\ninlist=%s" % repr(inlist))
|
|
if inlist:
|
|
self.log(5, "got one or more sockets with inbound data")
|
|
#self.nodesLock.acquire()
|
|
for sock in inlist:
|
|
node = self.nodeWhichOwnsSock(sock)
|
|
if node != None:
|
|
node._doRx()
|
|
#self.nodesLock.release()
|
|
|
|
elif self.fg:
|
|
return
|
|
|
|
else:
|
|
time.sleep(0.1)
|
|
except:
|
|
#self.nodesLock.release()
|
|
traceback.print_exc()
|
|
self.log(1, "core handler thread crashed")
|
|
self.isRunning_rx = False
|
|
self.log(3, "core handler thread terminated")
|
|
|
|
</t>
|
|
<t tx="aum.20040804130347"># create an instance of _KCore
|
|
core = KCore()
|
|
|
|
</t>
|
|
<t tx="aum.20040804130722">def __del__(self):
|
|
"""
|
|
Clean up on delete
|
|
"""
|
|
self.log(3, "node dying: %s" % self.name)
|
|
|
|
try:
|
|
del _nodes[self.name]
|
|
except:
|
|
pass
|
|
|
|
self.stop()
|
|
|
|
</t>
|
|
<t tx="aum.20040804132053">def _doRx(self):
|
|
"""
|
|
Receives and handles one incoming packet
|
|
|
|
Returns True if a packet got handled, or False if timeout
|
|
"""
|
|
# get next packet
|
|
self.log(5, "%s seeking socket lock" % self.name)
|
|
#self.sockLock.acquire()
|
|
self.log(5, "%s got socket lock" % self.name)
|
|
try:
|
|
item = self.sock.recvfrom(-1)
|
|
except i2p.socket.BlockError:
|
|
#self.sockLock.release()
|
|
self.log(5, "%s released socket lock after timeout" % self.name)
|
|
if not runCore:
|
|
time.sleep(0.1)
|
|
return False
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(5, "%s released socket lock after exception" % self.name)
|
|
#self.sockLock.release()
|
|
return True
|
|
#self.sockLock.release()
|
|
self.log(5, "%s released socket lock normally" % self.name)
|
|
|
|
try:
|
|
(data, dest) = item
|
|
except ValueError:
|
|
self.log(3, "node %s: recvfrom returned no dest, possible spoof" \
|
|
% self.name)
|
|
data = item[0]
|
|
dest = None
|
|
|
|
# try to decode
|
|
try:
|
|
d = messageDecode(data)
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(3, "failed to unpickle incoming data for node %s" % \
|
|
self.name)
|
|
return True
|
|
|
|
# ditch if not a dict
|
|
if type(d) != type({}):
|
|
self.log(3, "node %s: decoded packet is not a dict" % self.name)
|
|
return True
|
|
|
|
# temporary workaround for sam socket bug
|
|
if dest == None:
|
|
if hasattr(d, 'has_key') and d.has_key('dest'):
|
|
dest = d['dest']
|
|
|
|
# try to find it in our store
|
|
peerObj = self._findPeer(dest)
|
|
if peerObj == None:
|
|
# previously unknown peer - add it to our store
|
|
peerObj = self.addref(dest)
|
|
else:
|
|
peerObj.justSeen() # already exists - refresh its timestamp
|
|
self.addref(peerObj.dest)
|
|
|
|
# drop packet if no msgId
|
|
msgId = d.get('msgId', None)
|
|
if msgId == None:
|
|
self.log(3, "no msgId, dropping")
|
|
return True
|
|
del d['msgId']
|
|
|
|
msgType = d.get('type', 'unknown')
|
|
|
|
if desperatelyDebugging:
|
|
pass
|
|
#set_trace()
|
|
|
|
# if a local RPC is awaiting this message, fire its callback
|
|
item = self.rpcBindings.get((peerObj.dest, msgId), None)
|
|
if item:
|
|
rpc, peer = item
|
|
try:
|
|
rpc.unbindPeerReply(peerObj, msgId)
|
|
if desperatelyDebugging:
|
|
set_trace()
|
|
rpc.on_reply(peerObj, msgId, **d)
|
|
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(2, "unhandled exception in RPC on_reply")
|
|
else:
|
|
# find a handler, fallback on 'unknown'
|
|
self.log(5, "\nnode %s\ngot msg id %s type %s:\n%s" % (
|
|
self.name, msgId, msgType, d))
|
|
hdlrName = d.get('type', 'unknown')
|
|
hdlr = getattr(self, "_on_"+hdlrName)
|
|
try:
|
|
if desperatelyDebugging:
|
|
set_trace()
|
|
hdlr(peerObj, msgId, **d)
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(2, "unhandled exception in unbound packet handler %s" % hdlrName)
|
|
|
|
return True
|
|
|
|
</t>
|
|
<t tx="aum.20040804132551">def nodeWhichOwnsSock(self, sock):
|
|
"""
|
|
returns ref to node which owns a socket
|
|
"""
|
|
for node in self.nodes:
|
|
if node.sock == sock:
|
|
return node
|
|
return None
|
|
</t>
|
|
<t tx="aum.20040804140615">class KTestNetwork(stasher.KBase):
|
|
"""
|
|
Builds and runs a variable-sized test network
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040804140958">def __init__(self, numnodes=numTestNodes, doPings=True, purge=False):
|
|
"""
|
|
Builds the test network
|
|
"""
|
|
global Socket, select
|
|
|
|
self.trigger = 0
|
|
|
|
stasher.core.n = self # for convenience while debugging
|
|
|
|
if purge:
|
|
self.purge()
|
|
|
|
if 0:
|
|
if KTestNetwork.aNetworkExists:
|
|
raise Exception("A test network already exists, may not create another")
|
|
KTestNetwork.aNetworkExists = True
|
|
|
|
self.nodes = []
|
|
|
|
for i in range(numnodes):
|
|
nodeName = "kademlia-testnode-%s" % i
|
|
self.log(3, "Creating test node %s" % nodeName)
|
|
node = KNode(nodeName, in_depth=0, out_depth=0)
|
|
node.idx = i
|
|
node.n = self
|
|
self.nodes.append(node)
|
|
#print node.peers
|
|
|
|
self.log(3, "test network successfully created")
|
|
|
|
</t>
|
|
<t tx="aum.20040804141700">def __getitem__(self, num):
|
|
"""
|
|
Allows test network to be treated as array, returns the 'num'th node
|
|
"""
|
|
return self.nodes[num]
|
|
|
|
</t>
|
|
<t tx="aum.20040804141700.1">def __len__(self):
|
|
return len(self.nodes)
|
|
|
|
</t>
|
|
<t tx="aum.20040804141801">def start(self, doPings=True):
|
|
"""
|
|
Starts up the test network
|
|
"""
|
|
for node in self.nodes:
|
|
self.log(3, "starting node %s" % node.name)
|
|
node.start(doPings)
|
|
|
|
</t>
|
|
<t tx="aum.20040804141824">def stop(self):
|
|
"""
|
|
Stops (or tries to stop) the test network
|
|
"""
|
|
for node in self.nodes:
|
|
self.log(3, "stopping node %s" % node.name)
|
|
node.stop()
|
|
|
|
</t>
|
|
<t tx="aum.20040804142640">aNetworkExists = False
|
|
</t>
|
|
<t tx="aum.20040804200916">def _doHousekeeping(self):
|
|
"""
|
|
Performs periodical housekeeping on this node.
|
|
|
|
Activities include:
|
|
- checking pending records for timeouts
|
|
"""
|
|
now = time.time()
|
|
|
|
# DEPRECATED - SWITCH TO RPC-based
|
|
# check for expired pings
|
|
for msgId, (dest, q, pingDeadline) in self.pendingPings.items():
|
|
|
|
if pingDeadline > now:
|
|
# not timed out, leave in pending
|
|
continue
|
|
|
|
# ping has timed out
|
|
del self.pendingPings[msgId]
|
|
q.put(False)
|
|
|
|
# check for timed-out RPCs
|
|
for rpc in self.rpcPending[:]:
|
|
if rpc.nextTickTime != None and now >= rpc.nextTickTime:
|
|
try:
|
|
rpc.on_tick()
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(2, "unhandled exception in RPC on_tick")
|
|
|
|
</t>
|
|
<t tx="aum.20040804204524">class KPendingResultBase:
|
|
"""
|
|
Class which holds the details of an RPC sent to another node.
|
|
"""
|
|
@others
|
|
|
|
</t>
|
|
<t tx="aum.20040804204524.1">def __init__(self, node, typ, **kw):
|
|
"""
|
|
Creates a pending result object
|
|
|
|
Arguments:
|
|
- node - node which is waiting for this result
|
|
- typ - operation type on which we're awaiting a response,
|
|
one of 'ping', 'findNode', 'findData', 'store'
|
|
|
|
Keywords:
|
|
- gotta think about this
|
|
"""
|
|
self.node = node
|
|
self.typ = type
|
|
|
|
# dict of msgId, peer pairs
|
|
self.msgIds = {}
|
|
|
|
# indicates when to timeout and return the best available result
|
|
self.deadline = time.time() + timeout[typ]
|
|
|
|
# add ourself to node
|
|
self.node.pendingResults.append(self)
|
|
|
|
</t>
|
|
<t tx="aum.20040804205057">def _doChug(self):
|
|
"""
|
|
Do what's needed to drive the node.
|
|
Handle incoming packets
|
|
Check on and action timeouts
|
|
"""
|
|
# handle all available packets
|
|
while self._doRx():
|
|
pass
|
|
|
|
# do maintenance - eg processing timeouts
|
|
self._doHousekeeping()
|
|
|
|
</t>
|
|
<t tx="aum.20040804205740">def __cmp__(self, other):
|
|
|
|
# for sorting pending results by deadline
|
|
return cmp(self.deadline, other.deadline)
|
|
|
|
</t>
|
|
<t tx="aum.20040804210421">def append(self, peer, msgId):
|
|
"""
|
|
Adds a (peer, msgId) pair, to watch out for
|
|
"""
|
|
peer = self.node._normalisePeer(peer)
|
|
|
|
self.msgIds[msgId] = peer
|
|
self.node.awaitingMsgIds[msgId] = self
|
|
|
|
</t>
|
|
<t tx="aum.20040804210911">def wait(self):
|
|
"""
|
|
Called by application-level routines to wait on and return some kind of result
|
|
"""
|
|
return self.queue.get()
|
|
</t>
|
|
<t tx="aum.20040804211058">def check(self, now=None):
|
|
"""
|
|
Checks for a timeout condition, which if one occurs, sticks the best
|
|
available result onto the queue to be picked up by the caller
|
|
"""
|
|
if now == None:
|
|
now = time.time()
|
|
if now > self.deadline:
|
|
self.queue.put(self.bestAvailableResult())
|
|
|
|
</t>
|
|
<t tx="aum.20040804211248">def on_tick(self):
|
|
"""
|
|
Override this in subclasses.
|
|
If a timeout occurs, this routine gets called, and should return
|
|
the 'best available result' to be delivered back to synchronous caller
|
|
"""
|
|
raise KNotImplemented
|
|
</t>
|
|
<t tx="aum.20040804211539">def on_packet(self, msgId, **details):
|
|
"""
|
|
Called when a packet of id msgId arrives
|
|
|
|
Should return True if this packet was for us, or False if not
|
|
"""
|
|
raise KNotImplemented
|
|
|
|
</t>
|
|
<t tx="aum.20040804212126">class KPendingResultPing(KPendingResultBase):
|
|
"""
|
|
for managing synchronous pings
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040804212126.1">def __init__(self, node):
|
|
|
|
KPendingResultBase.__init__(self, node, 'ping')
|
|
|
|
</t>
|
|
<t tx="aum.20040804212342">def on_tick(self):
|
|
"""
|
|
Handle synchronous ping timeouts
|
|
"""
|
|
return False
|
|
|
|
</t>
|
|
<t tx="aum.20040804212607">def on_packet(self, msgId, **details):
|
|
"""
|
|
Must have got back a ping reply
|
|
"""
|
|
self.destroySelf()
|
|
self.queue.put(True)
|
|
|
|
return True
|
|
</t>
|
|
<t tx="aum.20040804212714">def destroySelf(self):
|
|
"""
|
|
Remove ourself from node
|
|
"""
|
|
self.node.pendingResults.remove(self)
|
|
|
|
|
|
</t>
|
|
<t tx="aum.20040804213616">def __eq__(self, other):
|
|
|
|
#self.log(2, "KPeer: comparing %s to %s (%s to %s)" % (self, other, self.__class__, other.__class__))
|
|
res = self.id == getattr(other, 'id', None)
|
|
#self.log(2, "KPeer: res=%s" % res)
|
|
return res
|
|
|
|
</t>
|
|
<t tx="aum.20040804220932">def __ne__(self, other):
|
|
return not (self == other)
|
|
</t>
|
|
<t tx="aum.20040804223241"></t>
|
|
<t tx="aum.20040805001258">def execute(self):
|
|
"""
|
|
Only for synchronous (application-level) execution.
|
|
Wait for the RPC to complete (or time out) and return
|
|
whatever it came up with
|
|
"""
|
|
if core.fg:
|
|
print "servicing background thread"
|
|
while self.queue.empty():
|
|
core.cycle()
|
|
|
|
return self.queue.get()
|
|
|
|
</t>
|
|
<t tx="aum.20040805001449"></t>
|
|
<t tx="aum.20040805001742">def bindPeerReply(self, peer, msgId):
|
|
"""
|
|
Sets up the node to give us a callback when a reply
|
|
comes in from downstream peer 'peer' with msg id 'msgId'
|
|
"""
|
|
self.localNode.rpcBindings[(peer.dest, msgId)] = (self, peer)
|
|
|
|
</t>
|
|
<t tx="aum.20040805001926">def unbindPeerReply(self, peer, msgId):
|
|
"""
|
|
Disables the callback from node for replies
|
|
from peer 'peer' with msgId 'msgId'
|
|
"""
|
|
bindings = self.localNode.rpcBindings
|
|
peerdest = peer.dest
|
|
if bindings.has_key((peerdest, msgId)):
|
|
del bindings[(peerdest, msgId)]
|
|
|
|
</t>
|
|
<t tx="aum.20040805004949">def unbindAll(self):
|
|
"""
|
|
Remove all reply bindings
|
|
"""
|
|
bindings = self.localNode.rpcBindings
|
|
self.log(5, "node bindings before: %s" % bindings)
|
|
for k,v in bindings.items():
|
|
if v[0] == self:
|
|
del bindings[k]
|
|
self.log(5, "node bindings after: %s" % bindings)
|
|
|
|
</t>
|
|
<t tx="aum.20040805012557">class KRpc(KBase):
|
|
"""
|
|
Base class for RPCs between nodes.
|
|
Refer subclasses
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040805012557.1">def __init__(self, localNode, client=None, **kw):
|
|
"""
|
|
Holds all the information for an RPC
|
|
|
|
Arguments:
|
|
- localNode - the node from which this RPC is being driven
|
|
- client - a representation of who is initiating this rpc, one of:
|
|
- None - an API caller, which is to be blocked until the RPC completes
|
|
or times out
|
|
- (upstreamPeer, upstreamMsgId) - an upstream peer
|
|
- callable object - something which requires a callback upon completion
|
|
in which case the callable will be invoked with the RPC results as the
|
|
first argument
|
|
|
|
Keywords:
|
|
- cbArgs - optional - if given, and if client is a callback, the callback
|
|
will be invoked with the results as first argument, and this object as
|
|
second argument
|
|
"""
|
|
self.localNode = localNode
|
|
|
|
if client == None:
|
|
# an api client
|
|
self.isLocal = True
|
|
self.queue = Queue.Queue()
|
|
self.callback = None
|
|
elif callable(client):
|
|
self.isLocal = False
|
|
self.callback = client
|
|
elif isinstance(client, tuple):
|
|
# we're doing the RPC on behalf of an upstream peer
|
|
upstreamPeer, upstreamMsgId = client
|
|
upstreamPeer = localNode._normalisePeer(upstreamPeer)
|
|
self.isLocal = False
|
|
self.upstreamPeer = upstreamPeer
|
|
self.upstreamMsgId = upstreamMsgId
|
|
self.callback = None
|
|
|
|
# save keywords
|
|
self.__dict__.update(kw)
|
|
|
|
# set time for receiving a tick.
|
|
# if this is set to an int absolute time value, the on_tick method
|
|
# will be called as soon as possible after that time
|
|
self.nextTickTime = None
|
|
|
|
# and register with node as a pending command
|
|
self.localNode.rpcPending.append(self)
|
|
|
|
# now start up the request
|
|
self.start()
|
|
|
|
</t>
|
|
<t tx="aum.20040805013630">def start(self):
|
|
"""
|
|
Start the RPC running.
|
|
Override this in subclasses
|
|
"""
|
|
raise KNotImplemented
|
|
|
|
</t>
|
|
<t tx="aum.20040805013903">def on_reply(self, peer, msgId, **details):
|
|
"""
|
|
Callback which fires when a downstream peer replies
|
|
|
|
Override this in subclasses
|
|
"""
|
|
raise KNotImplemented
|
|
|
|
</t>
|
|
<t tx="aum.20040805013957">def on_tick(self):
|
|
"""
|
|
Callback which fires if the whole RPC times out, in which
|
|
case the RPC should return whatever it can
|
|
|
|
Override in subclasses
|
|
"""
|
|
self.localNode.rpcPending.remove(self)
|
|
|
|
</t>
|
|
<t tx="aum.20040805014353">class KRpcPing(KRpc):
|
|
"""
|
|
Implements the PING rpc as per Kademlia spec
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040805014353.1">def __init__(self, localNode, client=None, **kw):
|
|
"""
|
|
Creates and performs a PING RPC
|
|
|
|
Arguments:
|
|
- localNode - the node performing this RPC
|
|
- upstreamPeer - if given, the peer wanting a reply
|
|
- upstreamMsgId - if upstreamPeer is given, this is the msgId
|
|
of the RPC message from the upstream peer
|
|
|
|
Keywords:
|
|
- peer - the peer to ping - default is local node
|
|
"""
|
|
peer = kw.get('peer', None)
|
|
if peer != None:
|
|
peer = localNode._normalisePeer(peer)
|
|
self.peerToPing = peer
|
|
|
|
if kw.has_key('cbArgs'):
|
|
KRpc.__init__(self, localNode, client, cbArgs=kw['cbArgs'])
|
|
else:
|
|
KRpc.__init__(self, localNode, client)
|
|
|
|
</t>
|
|
<t tx="aum.20040805014900">def start(self):
|
|
"""
|
|
Sends out the ping
|
|
"""
|
|
peer = self.peerToPing
|
|
|
|
# are we ourselves being pinged?
|
|
if peer == None:
|
|
# yes, just reply
|
|
self.returnValue(True)
|
|
return
|
|
|
|
# no - we need to ping a peer
|
|
thisNode = self.localNode
|
|
|
|
msgId = thisNode.msgId = thisNode._msgIdAlloc()
|
|
|
|
# bind for peer response
|
|
self.bindPeerReply(peer, msgId)
|
|
|
|
# and send it off
|
|
self.log(3, "node %s sending ping" % self.localNode.name)
|
|
peer.send_ping(msgId=msgId)
|
|
|
|
# and set a reply timeout
|
|
self.nextTickTime = time.time() + timeout['ping']
|
|
|
|
</t>
|
|
<t tx="aum.20040805014900.1">def on_reply(self, peer, msgId, **details):
|
|
"""
|
|
Callback for PING reply
|
|
"""
|
|
self.log(3, "got ping reply from %s" % peer)
|
|
self.returnValue(True)
|
|
|
|
</t>
|
|
<t tx="aum.20040805030707">def terminate(self):
|
|
"""
|
|
Clean up after ourselves.
|
|
Mainly involves removing ourself from local node
|
|
"""
|
|
self.unbindAll()
|
|
try:
|
|
self.localNode.rpcPending.remove(self)
|
|
except:
|
|
#traceback.print_exc()
|
|
pass
|
|
|
|
</t>
|
|
<t tx="aum.20040805031135">def messageEncode(params):
|
|
"""
|
|
Serialise the dict 'params' for sending
|
|
|
|
Temporarily using bencode - replace later with a more
|
|
efficient struct-based impl.
|
|
"""
|
|
try:
|
|
return bencode.bencode(params)
|
|
except:
|
|
log(1, "encoder failed to encode: %s" % repr(params))
|
|
raise
|
|
|
|
</t>
|
|
<t tx="aum.20040805031135.1">def messageDecode(raw):
|
|
return bencode.bdecode(raw)
|
|
</t>
|
|
<t tx="aum.20040805032351">def on_tick(self):
|
|
"""
|
|
'tick' handler.
|
|
|
|
For PING RPC, the only time we should get a tick is when the ping
|
|
has timed out
|
|
"""
|
|
self.log(3, "timeout awaiting ping reply from %s" % self.peerToPing)
|
|
self.returnValue(False)
|
|
|
|
</t>
|
|
<t tx="aum.20040805040237">def _on_reply(self, peer, msgId, **kw):
|
|
"""
|
|
This should never happen
|
|
"""
|
|
self.log(4, "got unhandled reply:\npeer=%s\nmsgId=%s\nkw=%s" % (
|
|
peer, msgId, kw))
|
|
|
|
</t>
|
|
<t tx="aum.20040805040409">def send_reply(self, **kw):
|
|
"""
|
|
Sends an RPC reply back to upstream peer
|
|
"""
|
|
self.log(5, "\nnode %s\nreplying to peer %s:\n%s" % (
|
|
self.node, self, kw))
|
|
self.send_raw(type="reply", **kw)
|
|
|
|
</t>
|
|
<t tx="aum.20040805123632">def threadHousekeeping(self):
|
|
"""
|
|
Periodically invoke nodes' housekeeping
|
|
"""
|
|
self.log(3, "\nnode housekeeping thread running")
|
|
try:
|
|
while self.isRunning:
|
|
#self.log(4, "calling nodes' housekeeping methods")
|
|
#self.nodesLock.acquire()
|
|
for node in self.nodes:
|
|
node._doHousekeeping()
|
|
#self.nodesLock.release()
|
|
time.sleep(1)
|
|
self.log(3, "\nnode housekeeping thread terminated")
|
|
except:
|
|
#self.nodesLock.release()
|
|
traceback.print_exc()
|
|
self.log(1, "\nnode housekeeping thread crashed")
|
|
|
|
</t>
|
|
<t tx="aum.20040805130159">class KRpcFindNode(KRpc):
|
|
"""
|
|
Implements the FIND_NODE rpc as per Kademlia spec
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040805130514">def __init__(self, localNode, client=None, **kw):
|
|
"""
|
|
Creates and launches the findNode rpc
|
|
|
|
Arguments:
|
|
- localNode - the node performing this RPC
|
|
- client - see KRpc.__init__
|
|
|
|
Keywords:
|
|
- hash - a string, long int or KHash object representing
|
|
what we're looking for. treatment depends on type:
|
|
- KHash object - used as is
|
|
- string - gets wrapped into a KHash object
|
|
- long int - wrapped into a KHash object
|
|
refer KHash.__init__
|
|
- raw - whether 'hash' is already a hash, default True
|
|
- local - True/False - whether to only search local store,
|
|
or pass on the query to the network, default True
|
|
"""
|
|
kw = dict(kw)
|
|
if kw.get('raw', False):
|
|
h = kw['hash']
|
|
del kw['hash']
|
|
kw['raw'] = h
|
|
self.hashWanted = KHash(**kw)
|
|
else:
|
|
self.hashWanted = KHash(kw['hash'], **kw)
|
|
self.isLocalOnly = kw.get('local', True)
|
|
|
|
self.numQueriesPending = 0
|
|
|
|
self.numRounds = 0 # count number of rounds
|
|
self.numReplies = 0 # number of query replies received
|
|
self.numQueriesSent = 0
|
|
self.numPeersRecommended = 0
|
|
|
|
# whichever mode we're called from, we gotta find the k closest peers
|
|
self.localNode = localNode
|
|
self.peerTab = self.findClosestPeersInitial()
|
|
|
|
self.log(4, "KRpcFindNode: isLocalOnly=%s" % self.isLocalOnly)
|
|
|
|
if kw.has_key('cbArgs'):
|
|
KRpc.__init__(self, localNode, client, cbArgs=kw['cbArgs'])
|
|
else:
|
|
KRpc.__init__(self, localNode, client)
|
|
|
|
</t>
|
|
<t tx="aum.20040805131021.1">@
|
|
Tech overview:
|
|
- this implementation creates each Node ID as an SHA1 hash of
|
|
the node's 'destination' - the string which constitutes its
|
|
address as an I2P endpoint.
|
|
|
|
Datagram formats:
|
|
- each datagram sent from one node to another is a python dict object,
|
|
encoded and decoded with the 'bencode' object serialisation module.
|
|
- we use bencode because regular Python pickle is highly insecure,
|
|
allowing crackers to create malformed pickles which can have all
|
|
manner of detrimental effects, including execution of arbitrary code.
|
|
- the possible messages are listed below, along with their consituent
|
|
dictionary keys:
|
|
1. ping:
|
|
- msgId - a message identifier guaranteed to be unique
|
|
with respect to the sending node
|
|
2. findNode:
|
|
- msgId - unique message identifier
|
|
- hash - the hash we're looking for
|
|
- initiator - True/False, according to whether this node
|
|
should initiate/perform the findNode, or whether this
|
|
rpc is coming from another seeking node
|
|
3. findData:
|
|
- msgId - unique message identifier
|
|
- hash - the exact key hash of the data we want to retrieve
|
|
- initiator - True/False, according to whether this node
|
|
should initiate/perform the findNode, or whether this
|
|
rpc is coming from another seeking node
|
|
4. store:
|
|
- msgId - unique message identifier
|
|
- hash - the exact key hash of the data we want to store
|
|
- data - the data we want to store
|
|
5. reply:
|
|
- msgId - the original msgId we're replying to
|
|
The other items in a reply message depend on what kind
|
|
of message we're replying to, listed below:
|
|
1. ping - no additional data
|
|
2. findNode:
|
|
- nodes - a list of dests nearest the given hash
|
|
3. findData:
|
|
- nodes - as for findNode, OR
|
|
- data - the retrieved data, or None if not found
|
|
4. store:
|
|
- status - True or False according to whether
|
|
the store operation was successful
|
|
</t>
|
|
<t tx="aum.20040805140236">def _findnode(self, something=None, callback=None, **kw):
|
|
"""
|
|
Mainly for testing - does a findNode query on the network
|
|
|
|
Arguments:
|
|
- something - one of:
|
|
- plain string - the string gets hashed and used for the search
|
|
- int or long int - this gets used as the raw hash
|
|
- a KHash object - that's what gets used
|
|
- None - the value of the 'raw' keyword will be used instead
|
|
- callback - optional - if given, a callable object which will be
|
|
called upon completion, with the result as argument
|
|
|
|
Keywords:
|
|
- local - optional - if True, only returns the closest peers known to
|
|
node. if False, causes node to query other nodes.
|
|
default is False
|
|
- raw - one of:
|
|
- 20-byte string - this gets used as a binary hash
|
|
- 40-byte string - this gets used as a hex hash
|
|
"""
|
|
if not kw.has_key('local'):
|
|
kw = dict(kw)
|
|
kw['local'] = False
|
|
|
|
self.log(3, "about to instantiate findnode rpc")
|
|
if callback:
|
|
KRpcFindNode(self, callback, hash=something, **kw)
|
|
self.log(3, "asynchronously invoked findnode, expecting callback")
|
|
else:
|
|
lst = KRpcFindNode(self, hash=something, **kw).execute()
|
|
self.log(3, "back from findnode rpc")
|
|
res = [self._normalisePeer(p) for p in lst] # wrap in KPeer objects
|
|
return res
|
|
|
|
</t>
|
|
<t tx="aum.20040805140416">def start(self):
|
|
"""
|
|
Kicks off this RPC
|
|
"""
|
|
# if we're being called by an upstream initiator, just return the peer list
|
|
if self.isLocalOnly:
|
|
peerDests = [peer.dest for peer in self.peerTab]
|
|
self.log(5, "findNode: local only: returning to upstream with %s" % repr(peerDests))
|
|
self.returnValue(peerDests)
|
|
return
|
|
|
|
# just return nothing if we don't have any peers
|
|
if len(self.peerTab) == 0:
|
|
self.returnValue([])
|
|
return
|
|
|
|
# send off first round of queries
|
|
self.sendSomeQueries()
|
|
|
|
return
|
|
|
|
</t>
|
|
<t tx="aum.20040805140632">def on_reply(self, peer, msgId, **details):
|
|
"""
|
|
Callback for FIND_NODE reply
|
|
"""
|
|
# shorthand
|
|
peerTab = self.peerTab
|
|
|
|
self.numReplies += 1
|
|
|
|
# ------------------------------------------------------------
|
|
# determine who replied, and get the raw dests sent back
|
|
try:
|
|
peerRec = peerTab[peer]
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(3, "discarding findNode reply from unknown peer %s %s, discarding" % (
|
|
peer, details))
|
|
return
|
|
|
|
# one less query to wait for
|
|
self.numQueriesPending -= 1
|
|
|
|
# ----------------------------------------------------------
|
|
# peerRec is the peer that replied
|
|
# peers is a list of raw dests
|
|
|
|
# save ref to this peer, it's seemingly good
|
|
self.localNode.addref(peerRec.peer)
|
|
|
|
# mark it as having replied
|
|
if peerRec.state != 'queried':
|
|
self.log(2, "too weird - got a reply from a peer we didn't query")
|
|
peerRec.state = 'replied'
|
|
|
|
# wrap the returned peers as KPeer objects
|
|
peersReturned = details.get('result', [])
|
|
peersReturned = [self.localNode._normalisePeer(p) for p in peersReturned]
|
|
|
|
self.numPeersRecommended += len(peersReturned)
|
|
|
|
# and add them to table in state 'recommended'
|
|
for p in peersReturned:
|
|
peerTab.append(p, 'recommended')
|
|
|
|
# try to fire off more queries
|
|
self.sendSomeQueries()
|
|
|
|
# and check for and action possible end of query round
|
|
self.checkEndOfRound()
|
|
|
|
|
|
</t>
|
|
<t tx="aum.20040805141509">def on_tick(self):
|
|
"""
|
|
Callback for FIND_NODE reply timeout
|
|
"""
|
|
# check for timeouts, and update offending peers
|
|
now = time.time()
|
|
for peerRec in self.peerTab:
|
|
if peerRec.hasTimedOut(now):
|
|
peerRec.state = 'timeout'
|
|
|
|
# makes room for more queries
|
|
self.sendSomeQueries()
|
|
|
|
# possible end of round
|
|
self.checkEndOfRound()
|
|
|
|
# schedule next tick
|
|
self.nextTickTime = time.time() + 5
|
|
|
|
</t>
|
|
<t tx="aum.20040805143215">@
|
|
Verbatim extract from original Kademlia paper follows:
|
|
|
|
The lookup initiator starts by picking x nodes from its closest
|
|
non-empty k-bucket (or, if that bucket has fewer than x
|
|
entries, it just takes the closest x nodes it knows of).
|
|
|
|
The initiator then sends parallel, asynchronous
|
|
FIND NODE RPCs to the x nodes it has chosen.
|
|
x is a system-wide concurrency parameter, such as 3.
|
|
|
|
In the recursive step, the initiator resends the
|
|
FIND NODE to nodes it has learned about from previous RPCs.
|
|
|
|
[Paraphrased - in the recursive step, the initiator sends a FIND_NODE to
|
|
each of the nodes that were returned as results of these previous
|
|
FIND_NODE RPCs.]
|
|
|
|
(This recursion can begin before all of the previous RPCs have
|
|
returned).
|
|
|
|
Of the k nodes the initiator has heard of closest to
|
|
the target, it picks x that it has not yet queried and resends
|
|
the FIND_NODE RPC to them.
|
|
|
|
Nodes that fail to respond quickly are removed from consideration
|
|
until and unless they do respond.
|
|
|
|
If a round of FIND_NODEs fails to return a node any closer
|
|
than the closest already seen, the initiator resends
|
|
the FIND NODE to all of the k closest nodes it has
|
|
not already queried.
|
|
|
|
The lookup terminates when the initiator has queried and gotten
|
|
responses from the k closest nodes it has seen.
|
|
</t>
|
|
<t tx="aum.20040805153146">class KRpcFindData(KRpcFindNode):
|
|
"""
|
|
variant of KRpcFindNode which returns key value if found
|
|
"""
|
|
@others
|
|
|
|
</t>
|
|
<t tx="aum.20040805153315">class KRpcStore(KRpc):
|
|
"""
|
|
Implements key storage
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040805153555">def _finddata(self, something=None, callback=None, **kw):
|
|
"""
|
|
As for findnode, but if data is found, return the data instead
|
|
"""
|
|
if not kw.has_key('local'):
|
|
kw = dict(kw)
|
|
kw['local'] = False
|
|
|
|
self.log(3, "about to instantiate finddata rpc")
|
|
if callback:
|
|
KRpcFindData(self, callback, hash=something, **kw)
|
|
self.log(3, "asynchronously invoked finddata, expecting callback")
|
|
else:
|
|
res = KRpcFindData(self, hash=something, **kw).execute()
|
|
self.log(3, "back from finddata rpc")
|
|
if not isinstance(res, str):
|
|
self.log(4, "findData RPC returned %s" % repr(res))
|
|
res = [self._normalisePeer(p) for p in res] # wrap in KPeer objects
|
|
return res
|
|
|
|
</t>
|
|
<t tx="aum.20040805153555.1">def _store(self, key, value, callback=None, **kw):
|
|
"""
|
|
Performs a STORE rpc
|
|
|
|
Arguments:
|
|
- key - string - text name of key
|
|
- value - string - value to store
|
|
|
|
Keywords:
|
|
- local - if given and true, only store value onto local store
|
|
"""
|
|
if not kw.has_key('local'):
|
|
kw = dict(kw)
|
|
kw['local'] = False
|
|
|
|
key = shahash(key)
|
|
if callback:
|
|
KRpcStore(self, callback, key=key, value=value, **kw)
|
|
self.log(3, "asynchronously invoked findnode, expecting callback")
|
|
else:
|
|
res = KRpcStore(self, key=key, value=value, **kw).execute()
|
|
return res
|
|
|
|
</t>
|
|
<t tx="aum.20040805154232">type = 'unknown' # override in subclass
|
|
|
|
</t>
|
|
<t tx="aum.20040805154253">type = 'ping'
|
|
|
|
</t>
|
|
<t tx="aum.20040805154306">type = 'findNode'
|
|
</t>
|
|
<t tx="aum.20040805154321">type = 'findData'
|
|
</t>
|
|
<t tx="aum.20040805154344">type = 'store'
|
|
</t>
|
|
<t tx="aum.20040805155412">@ignore
|
|
@language python
|
|
"""
|
|
Bloom filters in Python
|
|
Adam Langley <agl@imperialviolet.org>
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040805155412.1">import array
|
|
import struct
|
|
|
|
</t>
|
|
<t tx="aum.20040805155412.2">__all__ = ['Bloom']
|
|
|
|
mixarray = array.array ('B', '\x00' * 256)
|
|
# The mixarray is based on RC4 and used as diffusion in the hashing function
|
|
|
|
</t>
|
|
<t tx="aum.20040805155412.3">def mixarray_init (mixarray):
|
|
for i in range (256):
|
|
mixarray[i] = i
|
|
k = 7
|
|
for j in range (4):
|
|
for i in range (256):
|
|
s = mixarray[i]
|
|
k = (k + s) % 256
|
|
mixarray[i] = mixarray[k]
|
|
mixarray[k] = s
|
|
|
|
mixarray_init(mixarray)
|
|
|
|
</t>
|
|
<t tx="aum.20040805155412.4">class Bloom (object):
|
|
"""
|
|
Bloom filters provide a fast and compact way of checking set membership.
|
|
They do this by introducing a risk of a false positive (but there are
|
|
no false negatives).
|
|
|
|
For more information see:
|
|
- http://www.cs.wisc.edu/~cao/papers/summary-cache/node8.html
|
|
"""
|
|
@others
|
|
|
|
</t>
|
|
<t tx="aum.20040805155412.5">def __init__ (self, bytes, hashes):
|
|
'''
|
|
bytes is the size of the bloom filter in 8-bit bytes and
|
|
hashes is the number of hash functions to use.
|
|
Consult the web page linked above for values to use.
|
|
If in doubt, bytes = num_elements and hashes = 4
|
|
'''
|
|
self.hashes = hashes
|
|
self.bytes = bytes
|
|
|
|
self.a = self._make_array (bytes)
|
|
|
|
</t>
|
|
<t tx="aum.20040805155412.6">def _make_array (self, size):
|
|
|
|
a = array.array ('B')
|
|
# stupidly, there's no good way that I can see of
|
|
# resizing an array without allocing a huge string to do so
|
|
# thus I use this, slightly odd, method:
|
|
blocklen = 256
|
|
arrayblock = array.array ('B', '\x00' * blocklen)
|
|
todo = size
|
|
while (todo >= blocklen):
|
|
a.extend (arrayblock)
|
|
todo -= blocklen
|
|
if todo:
|
|
a.extend (array.array ('B', '\x00' * todo))
|
|
|
|
# now a is of the right length
|
|
return a
|
|
|
|
</t>
|
|
<t tx="aum.20040805155412.7">def _hashfunc (self, n, val):
|
|
'''Apply the nth hash function'''
|
|
|
|
global mixarray
|
|
|
|
b = [ord(x) for x in struct.pack ('I', val)]
|
|
c = array.array ('B', [0, 0, 0, 0])
|
|
for i in range (4):
|
|
c[i] = mixarray[(b[i] + n) % 256]
|
|
|
|
return struct.unpack ('I', c.tostring())[0]
|
|
|
|
</t>
|
|
<t tx="aum.20040805155412.8">def insert(self, val):
|
|
|
|
for i in range(self.hashes):
|
|
n = self._hashfunc(i, val) % (self.bytes * 8)
|
|
self.a[n // 8] |= self.bitmask[n % 8]
|
|
|
|
</t>
|
|
<t tx="aum.20040805155412.9">def __contains__ (self, val):
|
|
|
|
for i in range (self.hashes):
|
|
n = self._hashfunc (i, val) % (self.bytes * 8)
|
|
if not self.a[n // 8] & self.bitmask[n % 8]:
|
|
return 0
|
|
|
|
return 1
|
|
|
|
</t>
|
|
<t tx="aum.20040805155412.10">class CountedBloom (Bloom):
|
|
"""
|
|
Just like a Bloom filter, but provides counting (e.g. you can delete as well).
|
|
This uses 4 bits per bucket, so is generally four times larger
|
|
than the same non-counted bloom filter.
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040805155412.11">def __init__ (self, buckets, hashes):
|
|
'''
|
|
Please note that @buckets must be even.
|
|
Also note that with a Bloom object you give the
|
|
number of *bytes* and each byte is 8 buckets.
|
|
Here you're giving the number of buckets.
|
|
'''
|
|
assert buckets % 2 == 0
|
|
|
|
self.hashes = hashes
|
|
self.buckets = buckets
|
|
|
|
self.a = self._make_array (buckets // 2)
|
|
|
|
</t>
|
|
<t tx="aum.20040805155412.12">def insert (self, val):
|
|
|
|
masks = [(0x0f, 0xf0), (0xf0, 0x0f)]
|
|
shifts = [4, 0 ]
|
|
|
|
for i in range (self.hashes):
|
|
n = self._hashfunc (i, val) % self.buckets
|
|
byte = n // 2
|
|
bucket = n % 2
|
|
(notmask, mask) = masks[bucket]
|
|
shift = shifts[bucket]
|
|
bval = ((self.a[byte] & mask) >> shift)
|
|
if bval < 15:
|
|
# we shouldn't increment it if it's at the maximum
|
|
bval += 1
|
|
self.a[byte] = (self.a[byte] & notmask) | (bval << shift)
|
|
|
|
</t>
|
|
<t tx="aum.20040805155412.13">def __contains__ (self, val):
|
|
|
|
masks = [(0x0f, 0xf0), (0xf0, 0x0f)]
|
|
shifts = [4, 0 ]
|
|
|
|
for i in range (self.hashes):
|
|
n = self._hashfunc (i, val) % self.buckets
|
|
byte = n // 2
|
|
bucket = n % 2
|
|
(notmask, mask) = masks[bucket]
|
|
shift = shifts[bucket]
|
|
bval = ((self.a[byte] & mask) >> shift)
|
|
|
|
if bval == 0:
|
|
return 0
|
|
|
|
return 1
|
|
</t>
|
|
<t tx="aum.20040805155412.14">def __delitem__ (self, val):
|
|
|
|
masks = [(0x0f, 0xf0), (0xf0, 0x0f)]
|
|
shifts = [4, 0 ]
|
|
|
|
for i in range (self.hashes):
|
|
n = self._hashfunc (i, val) % self.buckets
|
|
byte = n // 2
|
|
bucket = n % 2
|
|
(notmask, mask) = masks[bucket]
|
|
shift = shifts[bucket]
|
|
bval = ((self.a[byte] & mask) >> shift)
|
|
|
|
if bval < 15: # we shouldn't decrement it if it's at the maximum
|
|
bval -= 1
|
|
|
|
self.a[byte] = (self.a[byte] & notmask) | (bval << shift)
|
|
|
|
</t>
|
|
<t tx="aum.20040805155412.15">if __name__ == '__main__':
|
|
|
|
print 'Testing bloom filter: there should be no assertion failures'
|
|
a = Bloom (3, 4)
|
|
|
|
a.insert (45)
|
|
print a.a
|
|
a.insert (17)
|
|
print a.a
|
|
a.insert (12)
|
|
print a.a
|
|
assert 45 in a
|
|
|
|
assert 45 in a
|
|
assert not 33 in a
|
|
assert 45 in a
|
|
assert 17 in a
|
|
assert 12 in a
|
|
|
|
c = 0
|
|
for x in range (255):
|
|
if x in a:
|
|
c += 1
|
|
print c
|
|
print float(c)/255
|
|
|
|
|
|
a = CountedBloom (24, 4)
|
|
a.insert (45)
|
|
print a.a
|
|
a.insert (17)
|
|
print a.a
|
|
a.insert (12)
|
|
print a.a
|
|
assert 45 in a
|
|
|
|
assert 45 in a
|
|
assert not 33 in a
|
|
assert 45 in a
|
|
assert 17 in a
|
|
assert 12 in a
|
|
|
|
c = 0
|
|
for x in range (255):
|
|
if x in a:
|
|
c += 1
|
|
print c
|
|
print float(c)/255
|
|
|
|
del a[45]
|
|
assert not 45 in a
|
|
|
|
</t>
|
|
<t tx="aum.20040805160344">bitmask = [0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01]
|
|
|
|
</t>
|
|
<t tx="aum.20040805182736">def __iter__(self):
|
|
return iter(self.nodes)
|
|
</t>
|
|
<t tx="aum.20040805185215">def findClosestPeersInitial(self):
|
|
"""
|
|
Searches our k-buckets, and returns a table of k of
|
|
peers closest to wanted hash into self.closestPeersInitial
|
|
"""
|
|
hashobj = self.hashWanted
|
|
|
|
lst = []
|
|
buckets = self.localNode.buckets
|
|
for bucket in buckets:
|
|
for peer in bucket:
|
|
lst.append(peer)
|
|
|
|
table = KPeerQueryTable(lst, self.hashWanted, 'start')
|
|
table.sort()
|
|
|
|
return table[:maxBucketSize]
|
|
|
|
</t>
|
|
<t tx="aum.20040805185902">def returnValue(self, res=None, **kw):
|
|
"""
|
|
Passes a return value back to the original caller, be it
|
|
the local application, or an upstream peer
|
|
|
|
Arguments:
|
|
- just one - a result object to pass back, if this RPC
|
|
was instigated by a local application call.
|
|
Note that if this RPC was instigated by an upstream
|
|
peer, this will be ignored.
|
|
|
|
Keywords:
|
|
- the items to return, in the case that this RPC was
|
|
instigated by an upstream peer. Ignored if this
|
|
RPC was instigated by a local application call.
|
|
Note - the RPC invocation/reply dict keys are
|
|
listed at the top of this source file.
|
|
"""
|
|
self.terminate()
|
|
if self.callback:
|
|
if hasattr(self, 'cbArgs'):
|
|
self.callback(res, self.cbArgs)
|
|
else:
|
|
self.callback(res)
|
|
elif self.isLocal:
|
|
self.queue.put(res)
|
|
else:
|
|
self.upstreamPeer.send_reply(msgId=self.upstreamMsgId,
|
|
**kw)
|
|
</t>
|
|
<t tx="aum.20040806002319">def addPeerIfCloser(self, peer):
|
|
"""
|
|
Maintains the private .peersToQuery array.
|
|
If the array is not yet maxed (ie, length < maxBucketSize),
|
|
the peer is simply added.
|
|
However, if the array is maxed, it finds the least-close peer,
|
|
and replaces it with the given peer if closer.
|
|
"""
|
|
</t>
|
|
<t tx="aum.20040806132808">def isCloserThanQueried(self, peer):
|
|
"""
|
|
Test function which returns True if argument 'peer'
|
|
is closer than all the peers in self.peersAlreadyQueried,
|
|
or False if not
|
|
"""
|
|
for p in self.peersAlreadyQueried:
|
|
if p.id.rawdistance(self.hashWanted) < peer.id.rawdistance(self.hashWanted):
|
|
return False
|
|
return True
|
|
|
|
</t>
|
|
<t tx="aum.20040806144708">def __del__(self):
|
|
|
|
#self.log(4, "\nRPC %s getting the chop" % (str(self)))
|
|
pass
|
|
|
|
</t>
|
|
<t tx="aum.20040806144812">def __str__(self):
|
|
|
|
return "<%s on node %s>" % (self.__class__.__name__, self.localNode.name)
|
|
|
|
</t>
|
|
<t tx="aum.20040806144829">def __repr__(self):
|
|
return str(self)
|
|
</t>
|
|
<t tx="aum.20040806223556">class KPeerQueryRecord(KBase):
|
|
"""
|
|
Keeps state information regarding a peer we're quering
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040806223857">class KPeerQueryTable(KBase):
|
|
"""
|
|
Holds zero or more instances of KPeerQuery and
|
|
presents/sorts table in different forms
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040806232701">def __str__(self):
|
|
return "<KTestNetwork: %d nodes>" % len(self.nodes)
|
|
</t>
|
|
<t tx="aum.20040806232714">def __repr__(self):
|
|
return str(self)
|
|
|
|
</t>
|
|
<t tx="aum.20040806234241">def sendSomeQueries(self, **kw):
|
|
"""
|
|
First step of findNode
|
|
|
|
Select alpha nodes that we haven't yet queried, and send them queries
|
|
"""
|
|
# bail if too busy
|
|
if self.numQueriesPending >= maxConcurrentQueries:
|
|
return
|
|
|
|
# shorthand
|
|
localNode = self.localNode
|
|
hashWanted = self.hashWanted
|
|
|
|
# randomly choose some peers
|
|
#somePeerRecs = self.peerTab.chooseN(numSearchPeers)
|
|
somePeerRecs = self.peerTab.select('start')
|
|
|
|
# start our ticker
|
|
self.nextTickTime = time.time() + timeout['findNode']
|
|
|
|
numQueriesSent = 0
|
|
|
|
# and send them findNode queries
|
|
if len(somePeerRecs) > 0:
|
|
for peerRec in somePeerRecs:
|
|
self.log(3, "querying %s" % peerRec)
|
|
if self.numQueriesPending < maxConcurrentQueries:
|
|
self.sendOneQuery(peerRec)
|
|
numQueriesSent += 1
|
|
else:
|
|
break
|
|
self.log(3, "%s queries sent, awaiting reply" % numQueriesSent)
|
|
else:
|
|
self.log(3, "no peer recs???")
|
|
for peerRec in self.peerTab:
|
|
self.log(4, "%s state=%s, dest=%s..." % (peerRec, peerRec.state, peerRec.dest[:12]))
|
|
|
|
</t>
|
|
<t tx="aum.20040807003220">def returnTheBestWeGot(self):
|
|
"""
|
|
Returns the k closest nodes to the wanted hash that we have
|
|
actually heard from
|
|
"""
|
|
# pick the peers which have replied to us
|
|
closest = self.peerTab.select('closest')
|
|
|
|
self.peerTab.dump()
|
|
|
|
# add ourself to the list - we could easily be one of the best
|
|
localNode = self.localNode
|
|
selfDest = localNode._normalisePeer(localNode.dest)
|
|
closest.append(selfDest, state='closest')
|
|
|
|
# sort in order of least distance first
|
|
closest.sort()
|
|
|
|
# pick the best k of these
|
|
#peersHeardFrom = peersHeardFrom[:maxBucketSize]
|
|
#peersHeardFrom = peersHeardFrom[:numSearchPeers]
|
|
|
|
# extract their dest strings
|
|
peers = [p.peer.dest for p in closest]
|
|
|
|
# pass these back
|
|
self.returnValue(peers)
|
|
|
|
# and we're done
|
|
return
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327">def __init__(self, lst=None, sorthash=None, state=None, **kw):
|
|
self.peers = []
|
|
if lst == None:
|
|
lst = []
|
|
else:
|
|
self.setlist(lst, state, **kw)
|
|
self.sorthash = sorthash
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.1">def setlist(self, lst, state=None, **kw):
|
|
for item in lst:
|
|
self.append(item, state, **kw)
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.2">def extend(self, items, state, **kw):
|
|
for item in items:
|
|
self.append(item, state, **kw)
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.3">def append(self, item, state=None, **kw):
|
|
|
|
if isinstance(item, KPeerQueryRecord):
|
|
self.log(5, "adding a KPeerQueryRecord, state=%s" % state)
|
|
if state != None:
|
|
item.state = state
|
|
item.__dict__.update(kw)
|
|
peerRec = item
|
|
|
|
elif isinstance(item, KPeer):
|
|
self.log(5, "adding a KPeer")
|
|
peerRec = KPeerQueryRecord(item, self, state, **kw)
|
|
|
|
else:
|
|
self.log(2, "bad peer %s" % repr(item))
|
|
raise KBadPeer
|
|
|
|
if peerRec not in self:
|
|
self.log(5, "peerRec=%s list=%s" % (peerRec, self.peers))
|
|
self.peers.append(peerRec)
|
|
else:
|
|
self.log(5, "trying to append duplicate peer???")
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.4">def remove(self, item):
|
|
self.peers.remove(item)
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.5">def getExpired(self):
|
|
"""
|
|
return a list of peers which have expired
|
|
"""
|
|
return KPeerQueryTable(
|
|
filter(lambda item: item.hasTimedOut(), self.peers),
|
|
self.sorthash
|
|
)
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.6">def purgeExpired(self):
|
|
"""
|
|
Eliminate peers which have expired
|
|
"""
|
|
for peer in self.peers:
|
|
if peer.hasTimedOut():
|
|
self.peers.remove(peer)
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.7">def __getitem__(self, idx):
|
|
"""
|
|
Allow the table to be indexed by any of:
|
|
- KPeerQueryRecord
|
|
- integer index
|
|
- long string - treated as dest
|
|
- short string - treated as peer id hash string
|
|
- KHash - finds peer with that id
|
|
- KPeer - returns peer with that peer
|
|
"""
|
|
if type(idx) == type(0):
|
|
return self.peers[idx]
|
|
elif isinstance(idx, KPeer):
|
|
for peer in self.peers:
|
|
if peer.peer == idx:
|
|
return peer
|
|
raise IndexError("Query table has no peer %s" % idx)
|
|
elif isinstance(idx, str):
|
|
if len(str) > 512:
|
|
for peer in self.peers:
|
|
if peer.peer.dest == idx:
|
|
return peer
|
|
raise IndexError("No peer with dest %s" % idx)
|
|
else:
|
|
for peer in self.peers:
|
|
if peer.peer.id.value == idx:
|
|
return peer
|
|
raise IndexError("No peer with dest hash %s" % idx)
|
|
elif isinstance(idx, KHash):
|
|
for peer in self.peers:
|
|
if peer.peer.id == idx:
|
|
return peer
|
|
raise IndexError("No peer with id %s" % idx)
|
|
else:
|
|
raise IndexError("Invalid selector %s" % repr(idx))
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.8">def __len__(self):
|
|
return len(self.peers)
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.9">def __getslice__(self, fromidx, toidx):
|
|
return KPeerQueryTable(self.peers[fromidx:toidx], self.sorthash)
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.10">def __iter__(self):
|
|
return iter(self.peers)
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.11">def sort(self):
|
|
"""
|
|
Sort the table in order of increasing distance from self.sorthash
|
|
"""
|
|
self.peers.sort()
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.12">def select(self, criterion):
|
|
"""
|
|
Returns a table of items for which criterion(item) returns True
|
|
Otherwise, if 'criterion' is a string, returns the items whose
|
|
state == criterion.
|
|
Otherwise, if 'criterion' is a list or tuple, return the items
|
|
whose state is one of the elements in criterion
|
|
"""
|
|
if callable(criterion):
|
|
func = criterion
|
|
elif type(criterion) in [type(()), type([])]:
|
|
func = lambda p: p.state in criterion
|
|
else:
|
|
func = lambda p: p.state == criterion
|
|
|
|
recs = []
|
|
for peerRec in self.peers:
|
|
if func(peerRec):
|
|
recs.append(peerRec)
|
|
return self.newtable(recs)
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.13">def filter(self, func):
|
|
"""
|
|
Eliminate, in place, all items where func(item) returns False
|
|
"""
|
|
for peerRec in self.peers:
|
|
if not func(peerRec):
|
|
self.peers.remove(peerRec)
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.14">def purge(self, func):
|
|
"""
|
|
Eliminate, in place, all items where func(item) returns True
|
|
"""
|
|
if 0 and desperatelyDebugging:
|
|
set_trace()
|
|
for peerRec in self.peers:
|
|
if func(peerRec):
|
|
self.peers.remove(peerRec)
|
|
|
|
</t>
|
|
<t tx="aum.20040807004327.15">def chooseN(self, n):
|
|
"""
|
|
Randomly select n peer query records
|
|
"""
|
|
candidates = self.peers[:]
|
|
|
|
self.log(3, "candidates = %s" % repr(candidates))
|
|
|
|
chosen = []
|
|
i = 0
|
|
|
|
if len(candidates) <= n:
|
|
chosen = candidates
|
|
else:
|
|
while i < n:
|
|
try:
|
|
peer = random.choice(candidates)
|
|
except:
|
|
self.log(2, "failed to choose one of %s" % repr(candidates))
|
|
raise
|
|
chosen.append(peer)
|
|
candidates.remove(peer)
|
|
i += 1
|
|
|
|
return self.newtable(chosen)
|
|
|
|
</t>
|
|
<t tx="aum.20040807004434">def __str__(self):
|
|
return "<KPeerQueryTable: %d peers>" % len(self) #.peers)
|
|
|
|
def __repr__(self):
|
|
return str(self)
|
|
|
|
</t>
|
|
<t tx="aum.20040807013038"></t>
|
|
<t tx="aum.20040807013038.1"></t>
|
|
<t tx="aum.20040807013038.2"></t>
|
|
<t tx="aum.20040807013038.3"></t>
|
|
<t tx="aum.20040807013038.4"></t>
|
|
<t tx="aum.20040807013038.5"></t>
|
|
<t tx="aum.20040807013038.6"></t>
|
|
<t tx="aum.20040807013038.7"></t>
|
|
<t tx="aum.20040807013038.8"></t>
|
|
<t tx="aum.20040807013038.9"></t>
|
|
<t tx="aum.20040807013411">def returnValue(self, items):
|
|
"""
|
|
override with a nicer call sig
|
|
"""
|
|
# a hack for testing - save this RPC object into the node
|
|
# so we can introspect it
|
|
self.localNode.lastrpc = self
|
|
|
|
items = items[:maxBucketSize]
|
|
|
|
self.reportStats()
|
|
|
|
KRpc.returnValue(self, items, result=items)
|
|
|
|
|
|
</t>
|
|
<t tx="aum.20040807013835">def newtable(self, items, state=None, **kw):
|
|
"""
|
|
Returns a new KPeerQueryTable object, based on this
|
|
one, but containing 'items'
|
|
"""
|
|
tab = KPeerQueryTable(items, sorthash=self.sorthash, state=state, **kw)
|
|
return tab
|
|
|
|
</t>
|
|
<t tx="aum.20040807013944"></t>
|
|
<t tx="aum.20040807014538">def __add__(self, other):
|
|
self.extend(other)
|
|
|
|
</t>
|
|
<t tx="aum.20040807033258">def __init__(self, peer, table, state=None, **kw):
|
|
|
|
self.peer = peer
|
|
self.dest = peer.dest
|
|
self.deadline = time.time() + timeout['findNode']
|
|
self.table = table
|
|
|
|
# state is always one of:
|
|
# - 'start' - have not yet sent query to peer
|
|
# - 'recommended' - peer was recommended by another peer, no query sent
|
|
# - 'queried' - sent query, awaiting reply or timeout
|
|
# - 'replied' - this peer has replied to our query
|
|
# - 'timeout' - timed out waiting for peer reply
|
|
# - 'toofar' - too far away to be of interest
|
|
# - 'closest' - this peer is one of the closest so far
|
|
|
|
if state == None:
|
|
state = 'start'
|
|
if not isinstance(state, str):
|
|
raise Exception("Invalid state %s" % state)
|
|
|
|
self.state = state
|
|
|
|
self.__dict__.update(kw)
|
|
|
|
</t>
|
|
<t tx="aum.20040807033258.1">def hasTimedOut(self, now=None):
|
|
if now == None:
|
|
now = time.time()
|
|
return self.state == 'queried' and now > self.deadline
|
|
|
|
</t>
|
|
<t tx="aum.20040807033258.2">def __cmp__(self, other):
|
|
|
|
return cmp(self.peer.id.rawdistance(self.table.sorthash),
|
|
other.peer.id.rawdistance(self.table.sorthash))
|
|
|
|
</t>
|
|
<t tx="aum.20040807033818">def __lt__(self, other):
|
|
return (cmp(self, other) < 0)
|
|
|
|
def __le__(self, other):
|
|
return (cmp(self, other) <= 0)
|
|
|
|
def __gt__(self, other):
|
|
return (cmp(self, other) > 0)
|
|
|
|
def __ge__(self, other):
|
|
return (cmp(self, other) <= 0)
|
|
|
|
</t>
|
|
<t tx="aum.20040807033818.1">def isCloserThanAllOf(self, tab):
|
|
"""
|
|
returns True if this peerRec is closer to the desired hash
|
|
than all of the peerRecs in table 'tab'
|
|
"""
|
|
if not isinstance(tab, KPeerQueryTable):
|
|
self.log(2, "invalid qtable %s" % repr(tab))
|
|
raise Exception("invalid qtable %s" % repr(tab))
|
|
|
|
for rec in tab:
|
|
if self > rec:
|
|
return False
|
|
return True
|
|
|
|
</t>
|
|
<t tx="aum.20040807034729">def isCloserThanOneOf(self, tab):
|
|
"""
|
|
returns True if this peerRec is closer to the desired hash
|
|
than one or more of of the peerRecs in table 'tab'
|
|
"""
|
|
if not isinstance(tab, KPeerQueryTable):
|
|
self.log(2, "invalid qtable %s" % repr(tab))
|
|
raise Exception("invalid qtable %s" % repr(tab))
|
|
|
|
for rec in tab:
|
|
if self < rec:
|
|
return True
|
|
return False
|
|
|
|
</t>
|
|
<t tx="aum.20040807044007">def __contains__(self, other):
|
|
self.log(5, "testing if %s is in %s" % (other, self.peers))
|
|
for peerRec in self.peers:
|
|
if peerRec.peer.dest == other.peer.dest:
|
|
return True
|
|
return False
|
|
|
|
</t>
|
|
<t tx="aum.20040807044007.1">def sendOneQuery(self, peerRec):
|
|
"""
|
|
Sends off a query to a single peer
|
|
"""
|
|
if peerRec.state != 'start':
|
|
self.log(2, "duh!! peer state %s:\n%s" % (peerRec.state, peerRec))
|
|
return
|
|
|
|
msgId = self.localNode._msgIdAlloc()
|
|
self.bindPeerReply(peerRec.peer, msgId)
|
|
peerRec.msgId = msgId
|
|
|
|
if self.type == 'findData':
|
|
peerRec.peer.send_findData(hash=self.hashWanted, msgId=msgId)
|
|
else:
|
|
peerRec.peer.send_findNode(hash=self.hashWanted, msgId=msgId)
|
|
|
|
peerRec.state = 'queried'
|
|
|
|
self.numQueriesPending += 1
|
|
|
|
self.numQueriesSent += 1
|
|
|
|
</t>
|
|
<t tx="aum.20040807052750">def run(self, func=None):
|
|
"""
|
|
Runs the core in foreground, with the client func in background
|
|
"""
|
|
if func==None:
|
|
func = test
|
|
|
|
self.bg = False
|
|
|
|
thread.start_new_thread(self.runClient, (func,))
|
|
|
|
set_trace()
|
|
|
|
self.threadRxPackets()
|
|
|
|
</t>
|
|
<t tx="aum.20040807053418">def stop(self):
|
|
self.isRunning = False
|
|
|
|
</t>
|
|
<t tx="aum.20040807114327">def runClient(self, func):
|
|
|
|
self.log(3, "Core: running client func")
|
|
try:
|
|
func()
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(3, "Core: client func exited")
|
|
self.stop()
|
|
</t>
|
|
<t tx="aum.20040808133629">def putKey(self, key, val, keyIsHashed=False):
|
|
"""
|
|
Stores a string into this storage under the key 'key'
|
|
|
|
Returns True if key was saved successfully, False if not
|
|
"""
|
|
try:
|
|
if keyIsHashed:
|
|
keyHashed = key
|
|
else:
|
|
keyHashed = shahash(key)
|
|
keyHashed = keyHashed.lower()
|
|
keyPath = os.path.join(self.keysDir, keyHashed)
|
|
file(keyPath, "wb").write(val)
|
|
self.log(4, "stored key: '%s'\nunder hash '%s'\n(keyIsHashed=%s)" % (
|
|
key, keyHashed, keyIsHashed))
|
|
return True
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(3, "failed to store key")
|
|
return False
|
|
|
|
</t>
|
|
<t tx="aum.20040808133629.1">def shahash(somestr, bin=False):
|
|
shaobj = sha.new(somestr)
|
|
if bin:
|
|
return shaobj.digest()
|
|
else:
|
|
return shaobj.hexdigest()
|
|
|
|
</t>
|
|
<t tx="aum.20040808133941">def getKey(self, key, keyIsHashed=False):
|
|
"""
|
|
Attempts to retrieve item from node's local file storage, which was
|
|
stored with key 'key'.
|
|
|
|
Returns value as a string if found, or None if not present
|
|
"""
|
|
try:
|
|
if keyIsHashed:
|
|
keyHashed = key
|
|
else:
|
|
keyHashed = shahash(key)
|
|
|
|
keyHashed = keyHashed.lower()
|
|
self.log(4, "key=%s, keyHashed=%s, keyIsHashed=%s" % (key, keyHashed, keyIsHashed))
|
|
|
|
keyPath = os.path.join(self.keysDir, keyHashed)
|
|
|
|
if os.path.isfile(keyPath):
|
|
return file(keyPath, "rb").read()
|
|
else:
|
|
return None
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(3, "error retrieving key '%s'" % key)
|
|
return None
|
|
|
|
</t>
|
|
<t tx="aum.20040808134739">def __init__(self, localNode, client=None, **kw):
|
|
"""
|
|
Creates and launches a STORE rpc
|
|
|
|
Arguments:
|
|
- localNode - the node performing this RPC
|
|
- client - see KRpc.__init__
|
|
|
|
Keywords:
|
|
- key - the key under which we wish to save the data
|
|
- value - the value we wish to save
|
|
- local - True/False:
|
|
- if True, only save in local store
|
|
- if False, do a findNode to find the nodes to save the
|
|
key to, and tell them to save it
|
|
default is True
|
|
"""
|
|
self.key = kw['key']
|
|
#self.keyHashed = shahash(self.key)
|
|
self.keyHashed = self.key
|
|
self.value = kw['value']
|
|
self.isLocalOnly = kw.get('local', True)
|
|
|
|
# set 'splitting' flag to indicate if we need to insert as splitfiles
|
|
self.splitting = len(self.value) > maxValueSize
|
|
|
|
self.log(4, "isLocalOnly=%s" % self.isLocalOnly)
|
|
|
|
if kw.has_key('cbArgs'):
|
|
KRpc.__init__(self, localNode, client, cbArgs=kw['cbArgs'])
|
|
else:
|
|
KRpc.__init__(self, localNode, client)
|
|
|
|
</t>
|
|
<t tx="aum.20040808134739.1">def start(self):
|
|
"""
|
|
Kicks off this RPC
|
|
"""
|
|
# if too big, then break up into <30k chunks
|
|
if self.splitting:
|
|
self.storeSplit()
|
|
return
|
|
|
|
# not too big - prefix a 0 chunk count, and go ahead as a single entity
|
|
self.value = "chunks:0\n" + self.value
|
|
|
|
# if local only, or no peers, just save locally
|
|
if self.isLocalOnly or len(self.localNode.peers) == 0:
|
|
result = self.localNode.storage.putKey(self.keyHashed, self.value, keyIsHashed=True)
|
|
if result:
|
|
result = 1
|
|
else:
|
|
result = 0
|
|
self.returnValue(result)
|
|
return
|
|
|
|
# no - se have to find peers to store the key to, and tell them to
|
|
# store the key
|
|
|
|
# launch a findNode rpc, continue in our callback
|
|
KRpcFindNode(self.localNode, self.on_doneFindNode,
|
|
hash=self.keyHashed, raw=True, local=False)
|
|
return
|
|
|
|
|
|
</t>
|
|
<t tx="aum.20040808135302">def on_doneFindNode(self, lst):
|
|
"""
|
|
Receive a callback from findNode
|
|
|
|
Send STORE command to each node that comes back
|
|
"""
|
|
localNode = self.localNode
|
|
|
|
# normalise results
|
|
normalisePeer = localNode._normalisePeer
|
|
peers = [normalisePeer(p) for p in lst] # wrap in KPeer objects
|
|
|
|
self.log(2, "STORE RPC findNode - got peers %s" % repr(peers))
|
|
|
|
i = 0
|
|
|
|
self.numPeersSucceeded = 0
|
|
self.numPeersFailed = 0
|
|
self.numPeersFinished = 0
|
|
|
|
# and fire off store messages for each peer
|
|
for peer in peers:
|
|
|
|
if peer.dest == localNode.dest:
|
|
self.log(3, "storing to ourself")
|
|
localNode.storage.putKey(self.keyHashed, self.value, keyIsHashed=True)
|
|
self.numPeersSucceeded += 1
|
|
self.numPeersFinished += 1
|
|
else:
|
|
msgId = self.localNode._msgIdAlloc()
|
|
self.log(4, "forwarding store cmd to peer:\npeer=%s\nmsgId=%s" % (peer, msgId))
|
|
self.bindPeerReply(peer, msgId)
|
|
peer.send_store(key=self.keyHashed, value=self.value, msgId=msgId)
|
|
i += 1
|
|
if i >= numStorePeers:
|
|
break
|
|
|
|
self.nextTickTime = time.time() + timeout['store']
|
|
|
|
self.log(2, "Sent store cmd to %s peers, awaiting responses" % i)
|
|
|
|
self.numPeersToStore = i
|
|
|
|
|
|
</t>
|
|
<t tx="aum.20040808140937">def returnValue(self, result):
|
|
"""
|
|
an override with a nicer call sig
|
|
"""
|
|
# a hack for testing - save this RPC object into the node
|
|
# so we can introspect it
|
|
self.localNode.lastrpc = self
|
|
|
|
try:
|
|
KRpc.returnValue(self, result, status=result)
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(3, "Failed to return %s" % repr(result))
|
|
KRpc.returnValue(self, 0, status=0)
|
|
|
|
</t>
|
|
<t tx="aum.20040808140937.1">def on_reply(self, peer, msgId, **details):
|
|
"""
|
|
callback which fires when we get a reply from a STORE we sent to a
|
|
peer
|
|
"""
|
|
self.numPeersSucceeded += 1
|
|
self.numPeersFinished += 1
|
|
|
|
if self.numPeersFinished == self.numPeersToStore:
|
|
# rpc is finished
|
|
self.returnValue(True)
|
|
|
|
</t>
|
|
<t tx="aum.20040808140937.2">def on_tick(self):
|
|
|
|
self.log(3, "Timeout awaiting store reply from %d out of %d peers" % (
|
|
self.numPeersToStore - self.numPeersSucceeded, self.numPeersToStore))
|
|
|
|
if self.numPeersSucceeded == 0:
|
|
self.log(3, "Store timeout - no peers replied, storing locally")
|
|
self.localNode.storage.putKey(self.keyHashed, self.value, keyIsHashed=True)
|
|
|
|
self.returnValue(True)
|
|
|
|
</t>
|
|
<t tx="aum.20040808142950">def _on_store(self, peer, msgId, **kw):
|
|
"""
|
|
Handles incoming STORE command
|
|
"""
|
|
self.log(4, "got STORE rpc from upstream:\npeer=%s\nmsgId=%s\nkw=%s" % (peer, msgId, kw))
|
|
|
|
KRpcStore(self, (peer, msgId), local=True, **kw)
|
|
|
|
</t>
|
|
<t tx="aum.20040808153427">def start(self):
|
|
"""
|
|
Kicks off the RPC.
|
|
If requested key is stored locally, simply returns it.
|
|
Otherwise, falls back on parent method
|
|
"""
|
|
# if we posses the data, just return the data
|
|
value = self.localNode.storage.getKey(self.hashWanted.asHex(), keyIsHashed=True)
|
|
if value != None:
|
|
self.log(4, "Found required value in local storage")
|
|
self.log(4, "VALUE='%s'" % value)
|
|
self.on_gotValue(value, self.hashWanted.asHex())
|
|
return
|
|
|
|
# no such luck - pass on to parent
|
|
KRpcFindNode.start(self)
|
|
|
|
</t>
|
|
<t tx="aum.20040808160207">def asHex(self):
|
|
return ("%040x" % self.value).lower()
|
|
|
|
</t>
|
|
<t tx="aum.20040808163651">def on_reply(self, peer, msgId, **details):
|
|
"""
|
|
Callback for FIND_NODE reply
|
|
"""
|
|
res = details.get('result', None)
|
|
if isinstance(res, str):
|
|
self.on_gotValue(res, self.hashWanted.asHex())
|
|
else:
|
|
KRpcFindNode.on_reply(self, peer, msgId, **details)
|
|
|
|
</t>
|
|
<t tx="aum.20040808203152">def __del__(self):
|
|
"""
|
|
Cleanup
|
|
"""
|
|
|
|
</t>
|
|
<t tx="aum.20040808213724">@first #! /usr/bin/env python
|
|
|
|
"""
|
|
A simple Hashcash implementation
|
|
|
|
Visit U{http://www.hashcash.org} for more info about
|
|
the theory and usage of hashcash.
|
|
|
|
Run this module through epydoc to get pretty doco.
|
|
|
|
Overview:
|
|
- implements a class L{HashCash}, with very configurable parameters
|
|
- offers two convenience wrapper functions, L{generate} and L{verify},
|
|
for those who can't be bothered instantiating a class
|
|
- given a string s, genToken produces a hashcash token
|
|
string t, as binary or base64
|
|
- generating t consumes a lot of cpu time
|
|
- verifying t against s is almost instantaneous
|
|
- this implementation produces clusters of tokens, to even out
|
|
the token generation time
|
|
|
|
Performance:
|
|
- this implementation is vulnerable to:
|
|
- people with lots of computers, especially big ones
|
|
- people writing bruteforcers in C (python is way slow)
|
|
- even with the smoothing effect of creating token clusters,
|
|
the time taken to create a token can vary by a factor of 7
|
|
|
|
Theory of this implementation:
|
|
|
|
- a hashcash token is created by a brute-force algorithm
|
|
of finding an n-bit partial hash collision
|
|
|
|
- given a string s, and a quality level q,
|
|
generate a 20-byte string h, such that:
|
|
|
|
1. h != s
|
|
2. len(h) == 20
|
|
3. ((sha(s) xor sha(h)) and (2 ^ q - 1)) == 0
|
|
|
|
- in other words, hash(h) and hash(s) have q least
|
|
significant bits in common
|
|
|
|
If you come up with a faster, but PURE PYTHON implementation,
|
|
using only modules included in standard python distribution,
|
|
please let me know so I can upgrade mine or link to yours.
|
|
|
|
Written by David McNab, August 2004
|
|
Released to the public domain.
|
|
"""
|
|
@others
|
|
|
|
</t>
|
|
<t tx="aum.20040808221610">import sha, array, random, base64, math
|
|
from random import randint
|
|
|
|
shanew = sha.new
|
|
|
|
</t>
|
|
<t tx="aum.20040808221610.1">def generate(value, quality, b64=False):
|
|
"""
|
|
Generates a hashcash token
|
|
|
|
This is a convenience wrapper function which saves you from having to
|
|
instantiate a HashCash object.
|
|
|
|
Arguments:
|
|
- value - a string against which to generate token
|
|
- quality - an int from 1 to 160 - typically values are 16 to 30
|
|
- b64 - if True, return the token as base64 (suitable for email,
|
|
news, and other text-based contexts), otherwise return a binary string
|
|
|
|
Quality values for desktop PC usage should typically be between 16 and 30.
|
|
Too low, and it makes an attacker's life easy.
|
|
Too high, and it makes life painful for the user.
|
|
"""
|
|
if b64:
|
|
format = 'base64'
|
|
else:
|
|
format = 'binary'
|
|
|
|
h = HashCash(quality=quality, format=format)
|
|
|
|
return h.generate(value)
|
|
|
|
</t>
|
|
<t tx="aum.20040808221610.3">def binify(L):
|
|
"""
|
|
Convert a python long int into a binary string
|
|
"""
|
|
res = []
|
|
while L:
|
|
res.append(chr(L & 0xFF))
|
|
L >>= 8
|
|
res.reverse()
|
|
return "".join(res)
|
|
|
|
</t>
|
|
<t tx="aum.20040808221610.4">def intify(s):
|
|
"""
|
|
Convert a binary string to a python long int
|
|
"""
|
|
n = 0L
|
|
for c in s:
|
|
n = (n << 8) | ord(c)
|
|
return n
|
|
|
|
</t>
|
|
<t tx="aum.20040808221925"># your own config settings - set these to get a good trade-off between
|
|
# token size and uniformity of time taken to generate tokens
|
|
#
|
|
# the final token size will be tokenSize * chunksPerToken for binary
|
|
# tokens, or ceil(4/3 * tokenSize * chunksPerToken) for base64 tokens
|
|
#
|
|
# the reason for building a token out of multiple token chunks is to
|
|
# try to even out the time taken for token generation
|
|
#
|
|
# without this, token generation time is very random, with some tokens
|
|
# generating almost instantaneously, and other tokens taking ages
|
|
|
|
defaultChunkSize = 3 # size of each chunk in a token
|
|
defaultNumChunks = 12 # number of chunks in each token
|
|
defaultQuality = 12 # number of partial hash collision bits required
|
|
defaultFormat = 'base64' # by default, return tokens in base64 format
|
|
defaultVerbosity = 0 # increase this to get more verbose output
|
|
|
|
</t>
|
|
<t tx="aum.20040808223205">def verify(value, quality, token):
|
|
"""
|
|
Verifies a hashcash token.
|
|
|
|
This is a convenience wrapper function which saves you from having to
|
|
instantiate a HashCash object.
|
|
|
|
Arguments:
|
|
- value - the string against which to check the hashcash token
|
|
- quality - the number of bits of token quality we require
|
|
- token - a hashcash token string
|
|
"""
|
|
h = HashCash(quality=quality)
|
|
|
|
return h.verify(value, token)
|
|
|
|
</t>
|
|
<t tx="aum.20040808224404">def test(nbits=14):
|
|
"""
|
|
Basic test function - perform encoding and decoding,
|
|
in plain and base64 formats, using the wrapper functions
|
|
"""
|
|
print "Test, using wrapper functions"
|
|
|
|
value = _randomString()
|
|
print "Generated random string\n%s" % value
|
|
print
|
|
|
|
print "Generating plain binary %s-bit token for:\n%s" % (nbits, value)
|
|
tok = generate(value, nbits)
|
|
|
|
print "Got token %s, now verifying" % repr(tok)
|
|
result = verify(value, nbits, tok)
|
|
|
|
print "Verify = %s" % repr(result)
|
|
print
|
|
|
|
print "Now generating base64 %s-bit token for:\n%s" % (nbits, value)
|
|
tok = generate(value, nbits, True)
|
|
|
|
print "Got base64 token %s, now verifying" % repr(tok)
|
|
result = verify(value, nbits, tok)
|
|
|
|
print "Verify = %s" % repr(result)
|
|
|
|
</t>
|
|
<t tx="aum.20040808231518"># get a boost of speed if psyco is available on target machine
|
|
try:
|
|
import psyco
|
|
psyco.bind(genToken)
|
|
psyco.bind(binify)
|
|
psyco.bind(intify)
|
|
except:
|
|
pass
|
|
|
|
</t>
|
|
<t tx="aum.20040809135303">class HashCash:
|
|
"""
|
|
Class for creating/verifying hashcash tokens
|
|
|
|
Feel free to subclass this, overriding the default attributes:
|
|
- chunksize
|
|
- numchunks
|
|
- quality
|
|
- format
|
|
- verbosity
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040809135303.1">def __init__(self, **kw):
|
|
"""
|
|
Create a HashCash object
|
|
|
|
Keywords:
|
|
- chunksize - size of each token chunk
|
|
- numchunks - number of chunks per token
|
|
- quality - strength of token, in bits:
|
|
- legal values are 1 to 160
|
|
- typical values are 10 to 30, larger values taking much
|
|
longer to generate
|
|
- format - 'base64' to output tokens in base64 format; any other
|
|
value causes tokens to be generated in binary string format
|
|
- verbosity - verbosity of output messages:
|
|
- 0 = silent
|
|
- 1 = critical only
|
|
- 2 = noisy
|
|
"""
|
|
for key in ['chunksize', 'numchunks', 'quality', 'format', 'verbosity']:
|
|
if kw.has_key(key):
|
|
setattr(self, key, kw[key])
|
|
|
|
self.b64ChunkLen = int(math.ceil(self.chunksize * 4.0 / 3))
|
|
|
|
</t>
|
|
<t tx="aum.20040809135303.2">def generate(self, value):
|
|
"""
|
|
Generate a hashcash token against string 'value'
|
|
"""
|
|
quality = self.quality
|
|
mask = 2 ** quality - 1
|
|
hV = sha.new(value).digest()
|
|
nHV = intify(hV)
|
|
|
|
maxTokInt = 2 ** (self.chunksize * 8)
|
|
|
|
tokenChunks = []
|
|
chunksPerToken = self.numchunks
|
|
|
|
# loop around generating random strings until we get one which,
|
|
# when xor'ed with value, produces a hash with the first n bits
|
|
# set to zero
|
|
while 1:
|
|
nTok = randint(0, maxTokInt)
|
|
sNTok = binify(nTok)
|
|
hSNTok = shanew(sNTok).digest()
|
|
nHSNTok = intify(hSNTok)
|
|
if (nHV ^ nHSNTok) & mask == 0:
|
|
# got a good token
|
|
if self.format == 'base64':
|
|
if not self._checkBase64(sNTok):
|
|
# chunk fails to encode/decode base64
|
|
if self.verbosity >= 2:
|
|
print "Ditching bad candidate token"
|
|
continue
|
|
bSNTok = self._enc64(sNTok)
|
|
if self.verbosity >= 2:
|
|
print "encoded %s to %s, expect chunklen %s" % (
|
|
repr(sNTok), repr(bSNTok), self.b64ChunkLen)
|
|
sNTok = bSNTok
|
|
# got something that works, add it to chunks, return if we got enough chunks
|
|
if sNTok in tokenChunks:
|
|
continue # already got this one
|
|
tokenChunks.append(sNTok)
|
|
if len(tokenChunks) == chunksPerToken:
|
|
return "".join(tokenChunks)
|
|
|
|
</t>
|
|
<t tx="aum.20040809135303.3">def verify(self, value, token):
|
|
"""
|
|
Verifies a hashcash token against string 'value'
|
|
"""
|
|
if self.verbosity >= 2:
|
|
print "Verify: checking token %s (len %s) against %s" % (token, len(token), value)
|
|
# mask is an int with least-significant 'q' bits set to 1
|
|
mask = 2 ** self.quality - 1
|
|
|
|
# breaking up token into its constituent chunks
|
|
chunks = []
|
|
|
|
# verify token size
|
|
if len(token) != self.chunksize * self.numchunks:
|
|
# try base64
|
|
decoded = False
|
|
try:
|
|
for i in range(0, self.numchunks):
|
|
b64chunk = token[(i * self.b64ChunkLen) : ((i + 1) * self.b64ChunkLen)]
|
|
chunk = self._dec64(b64chunk)
|
|
if len(chunk) != self.chunksize:
|
|
if self.verbosity >= 2:
|
|
print "Bad chunk length in decoded base64, wanted %s, got %s" % (
|
|
self.chunksize, len(chunk))
|
|
return False
|
|
chunks.append(chunk)
|
|
except:
|
|
if self.verbosity >= 2:
|
|
if decoded:
|
|
print "Bad token length"
|
|
else:
|
|
print "Base64 decode failed"
|
|
return False
|
|
else:
|
|
# break up token into its chunks
|
|
for i in range(0, self.numchunks):
|
|
chunks.append(token[(i * self.chunksize) : ((i + 1) * self.chunksize)])
|
|
|
|
# produce hash string and hash int for input string
|
|
hV = sha.new(value).digest()
|
|
nHv = intify(hV)
|
|
|
|
# test each chunk
|
|
if self.verbosity >= 2:
|
|
print "chunks = %s" % repr(chunks)
|
|
|
|
while chunks:
|
|
chunk = chunks.pop()
|
|
|
|
# defeat duplicate chunks
|
|
if chunk in chunks:
|
|
if self.verbosity >= 2:
|
|
print "Rejecting token chunk - duplicate exists"
|
|
return False
|
|
|
|
# hash the string and the token
|
|
hTok = sha.new(chunk).digest()
|
|
|
|
# defeat the obvious attack
|
|
if hTok == hV:
|
|
if self.verbosity >= 2:
|
|
print "Rejecting token chunk - equal to token"
|
|
return False
|
|
|
|
# test if these hashes have the least significant n bits in common
|
|
nHTok = intify(hTok)
|
|
if (nHTok ^ nHv) & mask != 0:
|
|
# chunk failed
|
|
if self.verbosity >= 2:
|
|
print "Rejecting token chunk %s - hash test failed" % repr(chunk)
|
|
return False
|
|
|
|
# pass
|
|
return True
|
|
|
|
</t>
|
|
<t tx="aum.20040809135512">def ctest(quality=14):
|
|
"""
|
|
Basic test function - perform token generation and verify, against
|
|
a random string. Instantiate a HashCash class instead of just using the
|
|
wrapper funcs.
|
|
"""
|
|
print "Test using HashCash class"
|
|
|
|
value = _randomString()
|
|
print "Generated random string\n%s" % value
|
|
print
|
|
|
|
hc = HashCash(quality=quality, format='base64')
|
|
|
|
print "Generating plain binary %s-bit token for:\n%s" % (quality, value)
|
|
tok = hc.generate(value)
|
|
|
|
print "Got token %s, now verifying" % repr(tok)
|
|
result = hc.verify(value, tok)
|
|
|
|
print "Verify = %s" % repr(result)
|
|
print
|
|
|
|
</t>
|
|
<t tx="aum.20040809135512.1">if __name__ == '__main__':
|
|
|
|
test()
|
|
|
|
</t>
|
|
<t tx="aum.20040809145111">def ntest():
|
|
"""
|
|
This function does 256 key generations in a row, and dumps
|
|
some statistical results
|
|
"""
|
|
# adjust these as desired
|
|
chunksize=3
|
|
numchunks=32
|
|
quality=6
|
|
numIterations = 256
|
|
|
|
import time
|
|
try:
|
|
import stats
|
|
except:
|
|
print "This test requires the stats module"
|
|
print "Get it (and its dependencies) from:"
|
|
print "http://www.nmr.mgh.harvard.edu/Neural_Systems_Group/gary/python.html"
|
|
return
|
|
|
|
print "Thrash test"
|
|
|
|
times = []
|
|
|
|
# create a hashcash token generator object
|
|
hc = HashCash(
|
|
chunksize=chunksize,
|
|
numchunks=numchunks,
|
|
quality=quality
|
|
)
|
|
|
|
# 256 times, create a random string and a matching hashcash token
|
|
for i in range(numIterations):
|
|
|
|
value = _randomString()
|
|
|
|
# measure time for a single token generation
|
|
then = time.time()
|
|
tok = hc.generate(value)
|
|
now = time.time()
|
|
times.append(now - then)
|
|
|
|
# sanity check, make sure it's valid
|
|
result = hc.verify(value, tok)
|
|
if not result:
|
|
print "Verify failed, token length=%s" % len(tok)
|
|
return
|
|
|
|
print "Generated %s of %s tokens" % (i, numIterations)
|
|
|
|
print "---------------------------------"
|
|
print "Thrash test performance results"
|
|
print "Token quality: %s bits" % quality
|
|
print "Min=%.3f max=%.3f max/min=%.3f mean=%.3f, median=%.3f, stdev=%.3f" % (
|
|
min(times),
|
|
max(times),
|
|
max(times)/min(times),
|
|
stats.lmean(times),
|
|
stats.lmedian(times),
|
|
stats.lstdev(times)
|
|
)
|
|
|
|
</t>
|
|
<t tx="aum.20040809145905">def _checkBase64(self, item):
|
|
"""
|
|
Ensures the item correctly encodes then decodes to/from base64
|
|
"""
|
|
#if self.verbose:
|
|
# print "Checking candidate token"
|
|
enc = self._enc64(item)
|
|
if len(enc) != self.b64ChunkLen:
|
|
if self.verbosity >= 1:
|
|
print "Bad candidate token"
|
|
return False
|
|
return self._dec64(enc) == item
|
|
|
|
</t>
|
|
<t tx="aum.20040809150949">def _enc64(self, item):
|
|
"""
|
|
Base64-encode a string, remove padding
|
|
"""
|
|
enc = base64.encodestring(item).strip()
|
|
while enc[-1] == '=':
|
|
enc = enc[:-1]
|
|
return enc
|
|
|
|
</t>
|
|
<t tx="aum.20040809150949.1">def _dec64(self, item):
|
|
"""
|
|
Base64-decode a string
|
|
"""
|
|
dec = base64.decodestring(item+"====")
|
|
return dec
|
|
|
|
</t>
|
|
<t tx="aum.20040809160231">def _randomString():
|
|
"""
|
|
For our tests below.
|
|
Generates a random-length human-readable random string,
|
|
between 16 and 80 chars
|
|
"""
|
|
chars = []
|
|
slen = randint(16, 80)
|
|
for i in range(slen):
|
|
chars.append(chr(randint(32, 128)))
|
|
value = "".join(chars)
|
|
return value
|
|
|
|
</t>
|
|
<t tx="aum.20040809162443"># override these at your pleasure
|
|
|
|
chunksize = defaultChunkSize
|
|
numchunks = defaultNumChunks
|
|
quality = defaultQuality
|
|
format = defaultFormat
|
|
verbosity = defaultVerbosity
|
|
|
|
</t>
|
|
<t tx="aum.20040809222157">def dump(self, detailed=0):
|
|
"""
|
|
Outputs a list of nodes and their connections
|
|
"""
|
|
if detailed:
|
|
self.dumplong()
|
|
return
|
|
|
|
for node in self.nodes:
|
|
print node.name + ":"
|
|
print " " + ", ".join([self.getPeerName(peer) for peer in node.peers])
|
|
|
|
</t>
|
|
<t tx="aum.20040810115020">def getPeerName(self, peer):
|
|
for n in self.nodes:
|
|
if n.dest == peer.dest:
|
|
return n.name
|
|
return "<??%s>" % n.dest[:8]
|
|
|
|
</t>
|
|
<t tx="aum.20040810122748">def whohas(self, key):
|
|
print "Nodes having key %s:" % key
|
|
|
|
h = KHash(key)
|
|
|
|
def hcmp(n1, n2):
|
|
res = cmp(h.rawdistance(n1.id), h.rawdistance(n2.id))
|
|
#print "compared: %s %s = %s" % (n1.idx, n2.idx, res)
|
|
return res
|
|
|
|
i = 0
|
|
|
|
nodes = self.nodes[:]
|
|
nodes.sort(hcmp)
|
|
|
|
for node in nodes:
|
|
if node.storage.getKey(key) != None:
|
|
i += 1
|
|
print "%3s" % node.idx,
|
|
if i % 16 == 0:
|
|
print
|
|
|
|
</t>
|
|
<t tx="aum.20040810125759">def whocanfind(self, key):
|
|
"""
|
|
Produces a list of nodes which can find key 'key'
|
|
"""
|
|
successes = []
|
|
failures = []
|
|
print "Nodes which can find key %s" % key
|
|
for i in range(len(self.nodes)):
|
|
node = self.nodes[i]
|
|
if node.get(key) != None:
|
|
print " %s found it" % node.name
|
|
successes.append(i)
|
|
else:
|
|
print " %s failed" % node.name
|
|
failures.append(i)
|
|
|
|
print "Successful finds: %s" % repr(successes)
|
|
print "Failed finds: %s" % repr(failures)
|
|
|
|
</t>
|
|
<t tx="aum.20040810131309">def closestto(self, key):
|
|
"""
|
|
Outputs a list of node names, in order of increasing 'distance'
|
|
from key
|
|
"""
|
|
key = KHash(key)
|
|
def nodecmp(node1, node2):
|
|
#self.log(3, "comparing node %s with %s" % (node1, node2))
|
|
return cmp(node1.id.rawdistance(key), node2.id.rawdistance(key))
|
|
|
|
nodes = self.nodes[:]
|
|
nodes.sort(nodecmp)
|
|
print "Nodes, in order of increasing distance from '%s'" % key
|
|
|
|
i = 0
|
|
for node in nodes:
|
|
i += 1
|
|
print "%3s" % node.idx,
|
|
if i % 16 == 0:
|
|
print
|
|
|
|
</t>
|
|
<t tx="aum.20040810132855">def findnode(self, idx, key):
|
|
"""
|
|
does a findnode on peer 'idx' against key 'key'
|
|
"""
|
|
peers = self.nodes[idx]._findnode(key)
|
|
print "Findnode on node %s (%s) returned:" % (self.nodes[idx].name, key)
|
|
|
|
peers = [self.getPeerIdx(p) for p in peers]
|
|
|
|
i = 0
|
|
for p in peers:
|
|
print "%3d" % p,
|
|
i += 1
|
|
if i % 16 == 0:
|
|
print
|
|
|
|
</t>
|
|
<t tx="aum.20040810141626">def dumpids(self):
|
|
print "Nodes listed by name and id"
|
|
for i in range(len(self.nodes)):
|
|
node = self.nodes[i]
|
|
print "%s: %s (%s...)" % (i, node.name, node.id.asHex()[:10])
|
|
</t>
|
|
<t tx="aum.20040810142448">def dumplong(self):
|
|
"""
|
|
Outputs a list of nodes and their connections
|
|
"""
|
|
for node in self.nodes:
|
|
print "%s: id=%s dest=%s" % (node.name, node.id.asHex()[:8], node.dest[:8])
|
|
for peer in node.peers:
|
|
npeer = self.getPeer(peer)
|
|
print " %s: id=%s dest=%s" % (npeer.name, npeer.id.asHex()[:8], npeer.dest[:8])
|
|
</t>
|
|
<t tx="aum.20040810142611">def getPeer(self, peer):
|
|
for n in self.nodes:
|
|
if n.dest == peer.dest:
|
|
return n
|
|
return None
|
|
|
|
</t>
|
|
<t tx="aum.20040810224601">def logexc(verbosity, msg, nPrev=0, clsname=None):
|
|
|
|
fd = StringIO("%s\n" % msg)
|
|
traceback.print_exc(file=fd)
|
|
log(verbosity, fd.getvalue(), nPrev, clsname)
|
|
|
|
</t>
|
|
<t tx="aum.20040811013733">class KTestSocket(stasher.KBase):
|
|
"""
|
|
Emulates an I2P Socket for testing
|
|
"""
|
|
# class-global mapping of b64 dests to sockets
|
|
opensocks = {}
|
|
totalQueuedItems = 0
|
|
|
|
def __init__(self, sessname, *args, **kw):
|
|
|
|
self.log(4, "creating simulated i2p socket %s" % sessname)
|
|
|
|
# that'll do for pseudo-random dests
|
|
self.dest = stasher.shahash(sessname) + "0" * 256
|
|
|
|
# set up our inbound queue
|
|
self.queue = Queue.Queue()
|
|
|
|
# register ourself
|
|
self.opensocks[self.dest] = self
|
|
|
|
def __del__(self):
|
|
|
|
# deregister ourself
|
|
del self.opensocks[self.dest]
|
|
|
|
def sendto(self, data, flags, dest):
|
|
|
|
self.opensocks[dest].queue.put((data, self.dest))
|
|
KTestSocket.totalQueuedItems += 1
|
|
|
|
def recvfrom(self, *args):
|
|
|
|
KTestSocket.totalQueuedItems -= 1
|
|
return self.queue.get()
|
|
|
|
def select(inlist, outlist, errlist, timeout=0):
|
|
|
|
log = stasher.log
|
|
log(5, "fake select called")
|
|
deadline = time.time() + timeout
|
|
while (time.time() < deadline) and KTestSocket.totalQueuedItems == 0:
|
|
time.sleep(0.1)
|
|
if KTestSocket.totalQueuedItems == 0:
|
|
return [], [], []
|
|
socksWithData = []
|
|
for sock in inlist:
|
|
if not sock.queue.empty():
|
|
socksWithData.append(sock)
|
|
log(5, "fake select returning %s" % repr(socksWithData))
|
|
return socksWithData, [], []
|
|
|
|
select = staticmethod(select)
|
|
|
|
def setblocking(self, val):
|
|
|
|
self.blocking = val
|
|
</t>
|
|
<t tx="aum.20040811111244"></t>
|
|
<t tx="aum.20040811111244.1">class KBase:
|
|
"""
|
|
A mixin which adds a class-specific logger
|
|
"""
|
|
def log(self, verbosity, msg):
|
|
|
|
log(verbosity, msg, 1, self.__class__.__name__)
|
|
|
|
def logexc(self, verbosity, msg):
|
|
|
|
logexc(verbosity, msg, 1, self.__class__.__name__)
|
|
|
|
</t>
|
|
<t tx="aum.20040811160223">def debug():
|
|
"""
|
|
Alternative testing entry point which runs the test() function
|
|
in background, and the engine in foreground, allowing the engine
|
|
to be stepped through with a debugger
|
|
"""
|
|
global desperatelyDebugging
|
|
desperatelyDebugging = True
|
|
core.run()
|
|
|
|
</t>
|
|
<t tx="aum.20040811162256"></t>
|
|
<t tx="aum.20040811162302">def on_reply(self, peer, msgId, **details):
|
|
"""
|
|
Callback for FIND_NODE reply
|
|
"""
|
|
# shorthand
|
|
peerRecs = self.peerRecs
|
|
|
|
# who replied?
|
|
try:
|
|
peerRec = peerRecs[peer]
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(3, "discarding findNode reply from unknown peer %s %s, discarding" % (
|
|
peer, details))
|
|
return
|
|
|
|
if logVerbosity >= 3:
|
|
try:
|
|
dests = "\n".join([d[:6] for d in details['nodes']])
|
|
except:
|
|
logexc(4, "*** find-node rpc reply = %s" % details)
|
|
|
|
self.log(3, "got findNode reply from %s:\n%s" % (peer, details))
|
|
self.unbindPeerReply(peer, msgId)
|
|
|
|
# save ref to this peer, it's seemingly good
|
|
self.localNode.addref(peerRec.peer)
|
|
|
|
# mark it as having replied
|
|
peerRec.state = 'replied'
|
|
|
|
# one less query to wait for
|
|
self.numQueriesPending -= 1
|
|
|
|
# save these as 'fromReply' peers
|
|
peersReturned = details.get('nodes', [])
|
|
peersReturned = [self.localNode._normalisePeer(p) for p in peersReturned]
|
|
peerRecsReturned = peerRecs.newtable(peersReturned, 'fromReply')
|
|
peerRecsReturned.sort()
|
|
peerRecsReturned.purge(lambda p:p in peerRecs or p.peer.dest == self.localNode.dest)
|
|
|
|
# update our node's KBucket
|
|
for peerObj in peersReturned:
|
|
dist = self.localNode.id.distance(peerObj.id)
|
|
self.localNode.buckets[dist].justSeenPeer(peerObj)
|
|
|
|
self.log(5, "peerRecsReturned = %s" % repr(peerRecsReturned))
|
|
|
|
if len(peerRecsReturned) > 0:
|
|
peerRecs.extend(peerRecsReturned, 'fromReply')
|
|
|
|
if desperatelyDebugging:
|
|
print "TRACING???"
|
|
set_trace()
|
|
|
|
# are there any peers we're still waiting to hear from?
|
|
if self.numQueriesPending == 0 and len(peerRecs.select(('idle', 'fromQuery'))) == 0:
|
|
|
|
# query round is finished - see how good the results are
|
|
repliedPeers = peerRecs.select('replied')
|
|
self.log(3, "====== query round finished, got %s" % repr(repliedPeers))
|
|
|
|
# if this round returned any peers better than the ones we've already
|
|
# heard from, then launch another round of queries
|
|
candidates = peerRecs.select('candidate')
|
|
ncandidates = len(candidates)
|
|
|
|
# test all returned peers, and see if one or more is nearer than
|
|
# our candidates
|
|
closerPeers = []
|
|
gotNearer = 0
|
|
for rec in peerRecs.select('fromReply'):
|
|
if ncandidates == 0 or rec.isCloserThanOneOf(candidates):
|
|
self.log(3, "Got a closer peer (or no candiates yet)")
|
|
gotNearer = 1
|
|
|
|
if gotNearer:
|
|
# mark replied peers as candidates
|
|
for rec in peerRecs.select('replied'):
|
|
rec.state = 'candidate'
|
|
pass
|
|
else:
|
|
# no - all queries are exhausted - it's the end of this round
|
|
self.log(3, "Query round returned no closer peers")
|
|
self.returnTheBestWeGot()
|
|
|
|
self.sendSomeQueries()
|
|
|
|
</t>
|
|
<t tx="aum.20040811163127">def count(self, *args):
|
|
"""
|
|
returns the number of records whose state is one of args
|
|
"""
|
|
count = 0
|
|
for rec in self.peers:
|
|
if rec.state in args:
|
|
count += 1
|
|
return count
|
|
|
|
</t>
|
|
<t tx="aum.20040811221318">def changeState(self, oldstate, newstate):
|
|
"""
|
|
for all recs of state 'oldstate', change their
|
|
state to 'newstate'
|
|
"""
|
|
for p in self.peers:
|
|
if p.state == oldstate:
|
|
p.state = newstate
|
|
</t>
|
|
<t tx="aum.20040811221628">def checkEndOfRound(self):
|
|
"""
|
|
Checks if we've hit the end of a query round.
|
|
If so, and if either:
|
|
- we've got some closer peers, OR
|
|
- we've heard from less than maxBucketSize peers,
|
|
fire off more queries
|
|
|
|
Otherwise, return the best available
|
|
"""
|
|
peerTab = self.peerTab
|
|
|
|
if core.fg:
|
|
set_trace()
|
|
|
|
# has this query round ended?
|
|
if peerTab.count('start', 'queried') > 0:
|
|
# not yet
|
|
return
|
|
|
|
self.log(2, "********** query round ended")
|
|
|
|
# ------------------------------------
|
|
# end of round processing
|
|
|
|
self.numRounds += 1
|
|
|
|
# did we get any closer to required hash?
|
|
if self.type == 'findData' \
|
|
or self.gotAnyCloser() \
|
|
or peerTab.count('closest') < maxBucketSize:
|
|
|
|
# yes - save these query results
|
|
self.log(4, "starting another round")
|
|
peerTab.changeState('replied', 'closest')
|
|
peerTab.changeState('recommended', 'start')
|
|
|
|
# cull the shortlist
|
|
self.log(2, "culling to k peers")
|
|
if peerTab.count('closest') > maxBucketSize:
|
|
peerTab.sort()
|
|
excess = peerTab.select('closest')[maxBucketSize:]
|
|
excess.changeState('closest', 'toofar')
|
|
pass
|
|
|
|
# and start up another round
|
|
self.sendSomeQueries()
|
|
|
|
# did anything launch?
|
|
if peerTab.count('start', 'queried') == 0:
|
|
# no - we're screwed
|
|
self.returnTheBestWeGot()
|
|
|
|
# done for now
|
|
return
|
|
|
|
</t>
|
|
<t tx="aum.20040811222934">def gotAnyCloser(self):
|
|
"""
|
|
Tests if any peer records in state 'recommended' or 'replied'
|
|
are nearer than the records in state 'closest'
|
|
"""
|
|
peerTab = self.peerTab
|
|
|
|
# get current closest peers
|
|
closest = peerTab.select('closest')
|
|
|
|
# if none yet, then this was just end of first round
|
|
if len(closest) == 0:
|
|
return True
|
|
|
|
# get the peers we're considering
|
|
#candidates = peerTab.select(('recommended', 'replied'))
|
|
candidates = peerTab.select('recommended')
|
|
|
|
# now test them
|
|
gotOneCloser = False
|
|
for c in candidates:
|
|
#if c.isCloserThanOneOf(closest):
|
|
if c.isCloserThanAllOf(closest):
|
|
return True
|
|
|
|
# none were closer
|
|
return False
|
|
|
|
</t>
|
|
<t tx="aum.20040811223457">def doput():
|
|
|
|
n[0].put('roses', 'red')
|
|
|
|
</t>
|
|
<t tx="aum.20040811235333">def dump(self):
|
|
|
|
c = self.count
|
|
self.log(2,
|
|
"PeerQueryTable stats:\n"
|
|
"start: %s\n"
|
|
"recommended: %s\n"
|
|
"queried: %s\n"
|
|
"replied: %s\n"
|
|
"timeout: %s\n"
|
|
"closest: %s\n"
|
|
"toofar: %s\n"
|
|
"TOTAL: %s\n" % (
|
|
c('start'),
|
|
c('recommended'),
|
|
c('queried'),
|
|
c('replied'),
|
|
c('timeout'),
|
|
c('closest'),
|
|
c('toofar'),
|
|
len(self.peers)))
|
|
|
|
#states = [p.state for p in self.peers]
|
|
#self.log(3, "PeerQueryTable states:\n%s" % states)
|
|
|
|
</t>
|
|
<t tx="aum.20040812013918">class KTestMap(stasher.KBase):
|
|
"""
|
|
Creates a random set of interconnections between nodes
|
|
in a test network
|
|
"""
|
|
path = "testnet.topology-%s"
|
|
|
|
def __init__(self, numnodes):
|
|
|
|
path = self.path % numnodes
|
|
if os.path.isfile(path):
|
|
self.load(numnodes)
|
|
return
|
|
|
|
self.log(2, "Creating new test topology of %s nodes" % numnodes)
|
|
|
|
self.refs = {} # key is nodenum, val is list of ref nodenums
|
|
self.numnodes = numnodes
|
|
i = 0
|
|
for i in range(1, numnodes):
|
|
|
|
# get a random number not equal to i
|
|
while 1:
|
|
ref = random.randrange(0, i)
|
|
if ref != i:
|
|
break
|
|
|
|
# add it
|
|
self.refs[i] = ref
|
|
|
|
# now point node 0 somewhere
|
|
self.refs[0] = random.randrange(1, i)
|
|
|
|
# and save it
|
|
self.save()
|
|
|
|
def __getitem__(self, idx):
|
|
"""
|
|
Returns the ref num for node index idx
|
|
"""
|
|
return self.refs[idx]
|
|
|
|
def dump(self):
|
|
nodes = self.refs.keys()
|
|
nodes.sort()
|
|
for n in nodes:
|
|
print ("%2s -> %2s" % (n, self.refs[n])),
|
|
if (n + 1) % 8 == 0:
|
|
print
|
|
else:
|
|
print "|",
|
|
|
|
def load(self, numnodes):
|
|
path = self.path % numnodes
|
|
encoded = file(path, "rb").read()
|
|
decoded = pickle.loads(encoded)
|
|
self.numnodes, self.refs = decoded
|
|
self.log(2, "Restored existing topology of %s nodes" % numnodes)
|
|
|
|
def save(self):
|
|
path = self.path % self.numnodes
|
|
encoded = pickle.dumps((self.numnodes, self.refs))
|
|
file(path, "wb").write(encoded)
|
|
|
|
</t>
|
|
<t tx="aum.20040812015208">def connect(self, topology=None):
|
|
"""
|
|
Connect these nodes together
|
|
"""
|
|
if topology:
|
|
self.map = topology
|
|
else:
|
|
self.map = KTestMap(len(self.nodes))
|
|
|
|
nodeIdxs = self.map.refs.keys()
|
|
nodeIdxs.sort()
|
|
|
|
for idx in nodeIdxs:
|
|
#print "Node %s, adding ref to node %s" % (idx, self.map[idx])
|
|
self[idx].addref(self[self.map[idx]])
|
|
|
|
</t>
|
|
<t tx="aum.20040812020746">def purge(self):
|
|
|
|
os.system("rm -rf ~/.i2pkademlia")
|
|
|
|
</t>
|
|
<t tx="aum.20040812110124">if __name__ == '__main__':
|
|
|
|
main()
|
|
|
|
</t>
|
|
<t tx="aum.20040812125235">def __del__(self):
|
|
|
|
pass
|
|
#KTestNetwork.aNetworkExists = False
|
|
|
|
</t>
|
|
<t tx="aum.20040812152603">def cycle(self):
|
|
|
|
self.fg = True
|
|
self.threadRxPackets()
|
|
|
|
</t>
|
|
<t tx="aum.20040812221935">def findpath(self, i0, i1):
|
|
"""
|
|
Tries to find a path from idx0 to idx1, printing out
|
|
the nodes along the way
|
|
"""
|
|
def _findpath(self, idx0, idx1, tried):
|
|
#print "seeking path from %s to %s" % (idx0, idx1)
|
|
n0 = self.nodes[idx0]
|
|
n1 = self.nodes[idx1]
|
|
|
|
n0peers = [self.getPeer(p) for p in n0.peers]
|
|
|
|
n0peerIdxs = [self.nodes.index(p) for p in n0peers]
|
|
|
|
possibles = []
|
|
for idx in n0peerIdxs:
|
|
if idx == idx1:
|
|
# success
|
|
return [idx1]
|
|
if idx not in tried:
|
|
possibles.append(idx)
|
|
|
|
if possibles == []:
|
|
return None
|
|
|
|
#print " possibles = %s" % repr(possibles)
|
|
|
|
for idx in possibles:
|
|
tried.append(idx)
|
|
res = _findpath(self, idx, idx1, tried)
|
|
if isinstance(res, list):
|
|
res.insert(0, idx)
|
|
return res
|
|
|
|
return None
|
|
|
|
res = _findpath(self, i0, i1, [i0])
|
|
|
|
if isinstance(res, list):
|
|
res.insert(0, i0)
|
|
|
|
return res
|
|
</t>
|
|
<t tx="aum.20040812230933">def testconnectivity(self):
|
|
"""
|
|
Ensures that every peer can reach every other peer
|
|
"""
|
|
nNodes = len(self.nodes)
|
|
|
|
for i in range(nNodes):
|
|
for j in range(nNodes):
|
|
if i != j and not self.findpath(i, j):
|
|
print "No route from node %s to node %s" % (i, j)
|
|
return False
|
|
print "Every node can reach every other node"
|
|
return True
|
|
|
|
</t>
|
|
<t tx="aum.20040813013718">def getPeerIdx(self, peer):
|
|
|
|
for i in range(len(self.nodes)):
|
|
n = self.nodes[i]
|
|
if n.dest == peer.dest:
|
|
return i
|
|
return None
|
|
|
|
</t>
|
|
<t tx="aum.20040813015858">def rawdistance(self, other):
|
|
"""
|
|
calculates the 'distance' between this hash and another hash,
|
|
and returns it raw as this xor other
|
|
"""
|
|
return self.value ^ other.value
|
|
|
|
</t>
|
|
<t tx="aum.20040813200853">def reportStats(self):
|
|
"""
|
|
Logs a stat dump of query outcome
|
|
"""
|
|
if self.isLocalOnly:
|
|
return
|
|
self.log(2,
|
|
"query terminated after %s rounds, %s queries, %s replies, %s recommendations" % (
|
|
(self.numRounds+1),
|
|
self.numQueriesSent,
|
|
(self.numReplies+1),
|
|
self.numPeersRecommended
|
|
)
|
|
)
|
|
</t>
|
|
<t tx="aum.20040813202242">@first #! /usr/bin/env python
|
|
|
|
"""
|
|
Implements a simulated kademlia test network
|
|
"""
|
|
|
|
@others
|
|
|
|
</t>
|
|
<t tx="aum.20040813202517"></t>
|
|
<t tx="aum.20040813202541">@others
|
|
</t>
|
|
<t tx="aum.20040813202541.1">if __name__ == '__main__':
|
|
|
|
print "starting test"
|
|
pass
|
|
test()
|
|
|
|
</t>
|
|
<t tx="aum.20040813203116">import sys, os, Queue, pickle, time
|
|
from pdb import set_trace
|
|
|
|
import stasher
|
|
|
|
|
|
</t>
|
|
<t tx="aum.20040813203339">class KCore(stasher.KCore):
|
|
"""
|
|
Override kademlia core to use simulated I2P sockets
|
|
"""
|
|
def select(self, inlist, outlist, errlist, timeout):
|
|
|
|
#print "dummy select"
|
|
return KTestSocket.select(inlist, outlist, errlist, timeout)
|
|
|
|
</t>
|
|
<t tx="aum.20040813203339.1">class KNode(stasher.KNode):
|
|
"""
|
|
Override kademlia node class to use simulated test socket
|
|
"""
|
|
SocketFactory = KTestSocket
|
|
|
|
</t>
|
|
<t tx="aum.20040813203621"># number of nodes to build in test network
|
|
numTestNodes = 100
|
|
|
|
stasher.logToSocket = 19199
|
|
</t>
|
|
<t tx="aum.20040813204308"></t>
|
|
<t tx="aum.20040813204308.1">SocketFactory = None # defaults to I2P socket
|
|
|
|
</t>
|
|
<t tx="aum.20040813204858">def select(self, inlist, outlist, errlist, timeout):
|
|
|
|
return i2p.select.select(inlist, outlist, errlist, timeout)
|
|
|
|
</t>
|
|
<t tx="aum.20040813211551">def main():
|
|
"""
|
|
Command line interface
|
|
"""
|
|
global samAddr, clientAddr, logVerbosity, dataDir
|
|
|
|
argv = sys.argv
|
|
argc = len(argv)
|
|
|
|
try:
|
|
opts, args = getopt.getopt(sys.argv[1:],
|
|
"h?vV:S:C:sd:fl",
|
|
['help', 'version', 'samaddr=', 'clientaddr=',
|
|
'verbosity=', 'status', 'datadir=', 'foreground',
|
|
'shortversion', 'localonly',
|
|
])
|
|
except:
|
|
traceback.print_exc(file=sys.stdout)
|
|
usage("You entered an invalid option")
|
|
|
|
daemonise = True
|
|
verbosity = 2
|
|
debug = False
|
|
foreground = False
|
|
localOnly = False
|
|
|
|
for opt, val in opts:
|
|
|
|
if opt in ['-h', '-?', '--help']:
|
|
usage(True)
|
|
|
|
elif opt in ['-v', '--version']:
|
|
print "Stasher version %s" % version
|
|
sys.exit(0)
|
|
|
|
elif opt in ['-V', '--verbosity']:
|
|
logVerbosity = int(val)
|
|
|
|
elif opt in ['-f', '--foreground']:
|
|
foreground = True
|
|
|
|
elif opt in ['-S', '--samaddr']:
|
|
samAddr = val
|
|
|
|
elif opt in ['-C', '--clientaddr']:
|
|
clientAddr = val
|
|
|
|
elif opt in ['-s', '--status']:
|
|
dumpStatus()
|
|
|
|
elif opt in ['-d', '--datadir']:
|
|
dataDir = val
|
|
|
|
elif opt == '--shortversion':
|
|
sys.stdout.write("%s" % version)
|
|
sys.stdout.flush()
|
|
sys.exit(0)
|
|
|
|
elif opt in ['-l', '--localonly']:
|
|
localOnly = True
|
|
|
|
#print "Debug - bailing"
|
|
#print repr(opts)
|
|
#print repr(args)
|
|
#sys.exit(0)
|
|
|
|
# Barf if no command given
|
|
if len(args) == 0:
|
|
err("No command given")
|
|
usage(0, 1)
|
|
|
|
cmd = args.pop(0)
|
|
argc = len(args)
|
|
|
|
#print "cmd=%s, args=%s" % (repr(cmd), repr(args))
|
|
|
|
if cmd not in ['help', '_start', 'start', 'stop',
|
|
'hello', 'get', 'put', 'addref', 'getref',
|
|
'pingall']:
|
|
err("Illegal command '%s'" % cmd)
|
|
usage(0, 1)
|
|
|
|
if cmd == 'help':
|
|
usage()
|
|
|
|
# dirty hack
|
|
if foreground and cmd == 'start':
|
|
cmd = '_start'
|
|
|
|
# magic undocumented command name - starts node, launches its client server,
|
|
# this should only happen if we're spawned from a 'start' command
|
|
if cmd == '_start':
|
|
if argc not in [0, 1]:
|
|
err("start: bad argument count")
|
|
usage()
|
|
if argc == 0:
|
|
nodeName = defaultNodename
|
|
else:
|
|
nodeName = args[0]
|
|
|
|
# create and serve a node
|
|
#set_trace()
|
|
node = KNode(nodeName)
|
|
node.start()
|
|
log(3, "Node %s launched, dest = %s" % (node.name, node.dest))
|
|
node.serve()
|
|
sys.exit(0)
|
|
|
|
if cmd == 'start':
|
|
if argc not in [0, 1]:
|
|
err("start: bad argument count")
|
|
usage()
|
|
if argc == 0:
|
|
nodeName = defaultNodename
|
|
else:
|
|
nodeName = args[0]
|
|
pidFile = nodePidfile(nodeName)
|
|
|
|
if os.path.exists(pidFile):
|
|
err(("Stasher node '%s' seems to be already running. If you are\n" % nodeName)
|
|
+"absolutely sure it's not running, please remove its pidfile:\n"
|
|
+pidFile+"\n")
|
|
sys.exit(1)
|
|
|
|
# spawn off a node
|
|
import stasher
|
|
pid = spawnproc(sys.argv[0], "-S", samAddr, "-C", clientAddr, "_start", nodeName)
|
|
file(pidFile, "wb").write("%s" % pid)
|
|
print "Launched stasher node as pid %s" % pid
|
|
print "Pidfile is %s" % pidFile
|
|
sys.exit(0)
|
|
|
|
if cmd == 'stop':
|
|
if argc not in [0, 1]:
|
|
err("stop: bad argument count")
|
|
usage()
|
|
if argc == 0:
|
|
nodeName = defaultNodename
|
|
else:
|
|
nodename = args[0]
|
|
|
|
pidFile = nodePidfile(nodeName)
|
|
|
|
if not os.path.isfile(pidFile):
|
|
err("Stasher node '%s' is not running - cannot kill\n" % nodeName)
|
|
sys.exit(1)
|
|
|
|
pid = int(file(pidFile, "rb").read())
|
|
try:
|
|
killproc(pid)
|
|
print "Killed stasher node (pid %s)" % pid
|
|
except:
|
|
print "Failed to kill node (pid %s)" % pid
|
|
os.unlink(pidFile)
|
|
sys.exit(0)
|
|
|
|
try:
|
|
client = KNodeClient()
|
|
except:
|
|
traceback.print_exc()
|
|
err("Node doesn't seem to be up, or reachable on %s" % clientAddr)
|
|
return
|
|
|
|
|
|
if cmd == 'hello':
|
|
err("Node seems fine")
|
|
sys.exit(0)
|
|
|
|
elif cmd == 'get':
|
|
if argc not in [1, 2]:
|
|
err("get: bad argument count")
|
|
usage()
|
|
|
|
key = args[0]
|
|
|
|
if argc == 2:
|
|
# try to open output file
|
|
path = args[1]
|
|
try:
|
|
outfile = file(path, "wb")
|
|
except:
|
|
err("Cannot open output file %s" % repr(path))
|
|
usage(0, 1)
|
|
else:
|
|
outfile = sys.stdout
|
|
|
|
if logVerbosity >= 3:
|
|
sys.stderr.write("Searching for key - may take up to %s seconds or more\n" % (
|
|
timeout['findData']))
|
|
res = client.get(key, local=localOnly)
|
|
if res == None:
|
|
err("Failed to retrieve '%s'" % key)
|
|
sys.exit(1)
|
|
else:
|
|
outfile.write(res)
|
|
outfile.flush()
|
|
outfile.close()
|
|
sys.exit(0)
|
|
|
|
elif cmd == 'put':
|
|
if argc not in [1, 2]:
|
|
err("put: bad argument count")
|
|
usage()
|
|
|
|
key = args[0]
|
|
|
|
if argc == 2:
|
|
# try to open input file
|
|
path = args[1]
|
|
try:
|
|
infile = file(path, "rb")
|
|
except:
|
|
err("Cannot open input file %s" % repr(path))
|
|
usage(0, 1)
|
|
else:
|
|
infile = sys.stdin
|
|
|
|
val = infile.read()
|
|
if len(val) > maxValueSize:
|
|
err("File is too big - please trim to %s" % maxValueSize)
|
|
|
|
if logVerbosity >= 3:
|
|
sys.stderr.write("Inserting key - may take up to %s seconds\n" % (
|
|
timeout['findNode'] + timeout['store']))
|
|
res = client.put(key, val, local=localOnly)
|
|
if res == None:
|
|
err("Failed to insert '%s'" % key)
|
|
sys.exit(1)
|
|
else:
|
|
sys.exit(0)
|
|
|
|
elif cmd == 'addref':
|
|
if argc not in [0, 1]:
|
|
err("addref: bad argument count")
|
|
usage()
|
|
|
|
if argc == 1:
|
|
# try to open input file
|
|
path = args[0]
|
|
try:
|
|
infile = file(path, "rb")
|
|
except:
|
|
err("Cannot open input file %s" % repr(path))
|
|
usage(0, 1)
|
|
else:
|
|
infile = sys.stdin
|
|
|
|
ref = infile.read()
|
|
|
|
res = client.addref(ref)
|
|
if res == None:
|
|
err("Failed to add ref")
|
|
sys.exit(1)
|
|
else:
|
|
sys.exit(0)
|
|
|
|
elif cmd == 'getref':
|
|
if argc not in [0, 1]:
|
|
err("getref: bad argument count")
|
|
usage()
|
|
|
|
res = client.getref()
|
|
|
|
if argc == 1:
|
|
# try to open output file
|
|
path = args[0]
|
|
try:
|
|
outfile = file(path, "wb")
|
|
except:
|
|
err("Cannot open output file %s" % repr(path))
|
|
usage(0, 1)
|
|
else:
|
|
outfile = sys.stdout
|
|
|
|
if res == None:
|
|
err("Failed to retrieve node ref")
|
|
sys.exit(1)
|
|
else:
|
|
outfile.write(res)
|
|
outfile.flush()
|
|
outfile.close()
|
|
sys.exit(0)
|
|
|
|
elif cmd == 'pingall':
|
|
if logVerbosity > 2:
|
|
print "Pinging all peers, waiting %s seconds for results" % timeout['ping']
|
|
res = client.pingall()
|
|
print res
|
|
sys.exit(0)
|
|
|
|
</t>
|
|
<t tx="aum.20040813211933">def test1():
|
|
"""
|
|
A torturous test
|
|
"""
|
|
</t>
|
|
<t tx="aum.20040813212609">def usage(detailed=False, ret=0):
|
|
|
|
print "Usage: %s <options> [<command> [<ars>...]]" % sys.argv[0]
|
|
if not detailed:
|
|
print "Type %s -h for help" % sys.argv[0]
|
|
sys.exit(ret)
|
|
|
|
print "This is stasher, distributed file storage network that runs"
|
|
print "atop the anonymising I2P network (http://www.i2p.net)"
|
|
print "Written by aum - August 2004"
|
|
print
|
|
print "Options:"
|
|
print " -h, --help - display this help"
|
|
print " -v, --version - print program version"
|
|
print " -V, --verbosity=n - verbosity, default 1, 1=quiet ... 4=noisy"
|
|
print " -S, --samaddr=host:port - host:port of I2P SAM port, "
|
|
print " default %s" % i2p.socket.samaddr
|
|
print " -C, --clientaddr=host:port - host:port for socket interface to listen on"
|
|
print " for clients, default %s" % clientAddr
|
|
print " -d, --datadir=dir - directory in which stasher files get written"
|
|
print " default is ~/.stasher"
|
|
print " -f, --foreground - only valid for 'start' cmd - runs the node"
|
|
print " in foreground without spawning - for debugging"
|
|
print " -l, --localonly - only valid for get/put - restricts the get/put"
|
|
print " operation to the local node only"
|
|
print
|
|
print "Commands:"
|
|
print " start [<nodename>]"
|
|
print " - launches a single node, which forks off and runs in background"
|
|
print " nodename is a short unique nodename, default is '%s'" % defaultNodename
|
|
print " stop [<nodename>]"
|
|
print " - terminates running node <nodename>"
|
|
print " get <keyname> [<file>]"
|
|
print " - attempts to retrieve key <keyname> from the network, saving"
|
|
print " to file <file> if given, or to stdout if not"
|
|
print " put <keyname> [<file>]"
|
|
print " - inserts key <keyname> into the network, taking its content"
|
|
print " from file <file> if given, otherwise reads content from stdin"
|
|
print " addref <file>"
|
|
print " - adds a new noderef to the node, taking the base64 noderef"
|
|
print " from file <file> if given, or from stdin"
|
|
print " (if you don't have any refs, visit http://stasher.i2p, or use"
|
|
print " the dest in the file aum.stasher in cvs)"
|
|
print " getref <file>"
|
|
print " - uplifts the running node's dest as base64, writing it to file"
|
|
print " <file> if given, or to stdout"
|
|
print " hello"
|
|
print " - checks that local node is running"
|
|
print " pingall"
|
|
print " - diagnostic tool - pings all peers, waits for replies or timeouts,"
|
|
print " reports results"
|
|
print " help"
|
|
print " - display this help"
|
|
print
|
|
|
|
sys.exit(0)
|
|
|
|
</t>
|
|
<t tx="aum.20040813232532"></t>
|
|
<t tx="aum.20040813232532.1">class KNodeServer(KBase, SocketServer.ThreadingMixIn, SocketServer.TCPServer):
|
|
"""
|
|
Listens for incoming socket connections
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040813232532.2">class KNodeReqHandler(KBase, SocketServer.StreamRequestHandler):
|
|
"""
|
|
Manages a single client connection
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040813232532.3">def handle(self):
|
|
"""
|
|
Conducts all conversation for a single req
|
|
"""
|
|
req = self.request
|
|
client = self.client_address
|
|
server = self.server
|
|
node = self.server.node
|
|
|
|
read = self.rfile.read
|
|
readline = self.rfile.readline
|
|
write = self.wfile.write
|
|
flush = self.wfile.flush
|
|
|
|
finish = self.finish
|
|
|
|
# start with a greeting
|
|
write("Stasher version %s ready\n" % version)
|
|
|
|
# get the command
|
|
line = readline().strip()
|
|
|
|
try:
|
|
cmd, args = re.split("\\s+", line, 1)
|
|
except:
|
|
cmd = line
|
|
args = ''
|
|
|
|
self.log(3, "cmd=%s args=%s" % (repr(cmd), repr(args)))
|
|
|
|
if cmd in ["get", "getlocal"]:
|
|
isLocal = cmd == "getlocal"
|
|
value = node.get(args, local=isLocal)
|
|
if value == None:
|
|
write("notfound\n")
|
|
else:
|
|
write("ok\n%s\n%s" % (len(value), value))
|
|
flush()
|
|
time.sleep(2)
|
|
finish()
|
|
return
|
|
|
|
elif cmd in ["put", "putlocal"]:
|
|
isLocal = cmd == "putlocal"
|
|
try:
|
|
size = int(readline())
|
|
value = read(size)
|
|
res = node.put(args, value, local=isLocal)
|
|
if res:
|
|
write("ok\n")
|
|
else:
|
|
write("failed\n")
|
|
flush()
|
|
except:
|
|
traceback.print_exc()
|
|
write("exception\n")
|
|
finish()
|
|
return
|
|
|
|
elif cmd == 'addref':
|
|
try:
|
|
res = node.addref(args, True)
|
|
if res:
|
|
write("ok\n")
|
|
else:
|
|
write("failed\n")
|
|
flush()
|
|
except:
|
|
traceback.print_exc()
|
|
write("exception\n")
|
|
finish()
|
|
return
|
|
|
|
elif cmd == 'getref':
|
|
res = node.dest
|
|
write("ok\n")
|
|
write("%s\n" % res)
|
|
flush()
|
|
time.sleep(1)
|
|
finish()
|
|
return
|
|
|
|
elif cmd == 'pingall':
|
|
res = node._pingall()
|
|
write(res+"\n")
|
|
finish()
|
|
return
|
|
|
|
elif cmd == "die":
|
|
server.isRunning = False
|
|
write("server terminated\n")
|
|
finish()
|
|
|
|
else:
|
|
write("unrecognisedcommand\n")
|
|
finish()
|
|
return
|
|
|
|
</t>
|
|
<t tx="aum.20040813233036">def __init__(self, node, addr=None):
|
|
|
|
if addr == None:
|
|
addr = clientAddr
|
|
|
|
self.isRunning = True
|
|
|
|
self.node = node
|
|
|
|
listenHost, listenPort = addr.split(":")
|
|
listenPort = int(listenPort)
|
|
self.listenPort = listenPort
|
|
SocketServer.TCPServer.__init__(self, (listenHost, listenPort), KNodeReqHandler)
|
|
|
|
</t>
|
|
<t tx="aum.20040813234004"></t>
|
|
<t tx="aum.20040813234004.1">def serve(self):
|
|
"""
|
|
makes this node listen on socket for incoming client
|
|
connections, and services these connections
|
|
"""
|
|
server = KNodeServer(self)
|
|
server.serve_forever()
|
|
|
|
</t>
|
|
<t tx="aum.20040813234214">def serve_forever(self):
|
|
|
|
print "awaiting client connections on port %s" % self.listenPort
|
|
while self.isRunning:
|
|
self.handle_request()
|
|
|
|
</t>
|
|
<t tx="aum.20040814001156">class KNodeClient(KBase):
|
|
"""
|
|
Talks to a KNodeServer over a socket
|
|
|
|
Subclass this to implement Stasher clients in Python
|
|
"""
|
|
@others
|
|
|
|
</t>
|
|
<t tx="aum.20040814001156.1">def __init__(self, address=clientAddr):
|
|
|
|
if type(address) in [type(()), type([])]:
|
|
self.host, self.port = clientAddr
|
|
else:
|
|
self.host, self.port = clientAddr.split(":")
|
|
self.port = int(self.port)
|
|
|
|
self.hello()
|
|
|
|
</t>
|
|
<t tx="aum.20040814001456">def connect(self):
|
|
|
|
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
self.sock.connect((self.host, self.port))
|
|
|
|
self.rfile = self.sock.makefile("rb")
|
|
self.read = self.rfile.read
|
|
self.readline = self.rfile.readline
|
|
self.wfile = self.sock.makefile("wb")
|
|
self.write = self.wfile.write
|
|
self.flush = self.wfile.flush
|
|
|
|
# read greeting
|
|
greeting = self.readline()
|
|
parts = re.split("\\s+", greeting)
|
|
if parts[0] != "Stasher":
|
|
self.close()
|
|
raise Exception("Not connected to valid stasher interface")
|
|
|
|
</t>
|
|
<t tx="aum.20040814001456.2">def get(self, key, **kw):
|
|
"""
|
|
sends a get command to stasher socket, and retrieves
|
|
and interprets result
|
|
|
|
Arguments:
|
|
- key - key to retrieve
|
|
|
|
Keywords:
|
|
- local - default False - if True, only looks in local storage
|
|
|
|
Returns key's value if found, or None if key not found
|
|
"""
|
|
if kw.get('local', False):
|
|
cmd = 'getlocal'
|
|
else:
|
|
cmd = 'get'
|
|
|
|
self.connect()
|
|
|
|
self.write("%s %s\n" % (cmd, key))
|
|
self.flush()
|
|
|
|
#print "waiting for resp line"
|
|
res = self.readline().strip()
|
|
|
|
if res == "ok":
|
|
size = int(self.readline())
|
|
val = self.read(size)
|
|
self.close()
|
|
return val
|
|
else:
|
|
self.close()
|
|
return None
|
|
|
|
</t>
|
|
<t tx="aum.20040814001522">def close(self):
|
|
|
|
self.rfile.close()
|
|
#self.wfile.close()
|
|
self.sock.close()
|
|
|
|
</t>
|
|
<t tx="aum.20040814001912">def hello(self):
|
|
|
|
self.connect()
|
|
self.close()
|
|
</t>
|
|
<t tx="aum.20040814002236">def put(self, key, val, **kw):
|
|
"""
|
|
Tells remote stasher port to insert a file into the network
|
|
|
|
Arguments:
|
|
- key - key to insert under
|
|
- val - value to insert under this key
|
|
|
|
Keywords:
|
|
- local - default False - if True, only looks in local storage
|
|
|
|
"""
|
|
if kw.get('local', False):
|
|
cmd = 'putlocal'
|
|
else:
|
|
cmd = 'put'
|
|
|
|
self.connect()
|
|
self.write("%s %s\n" % (cmd, key))
|
|
self.write("%s\n" % len(val))
|
|
self.write(val)
|
|
self.flush()
|
|
|
|
res = self.readline().strip()
|
|
|
|
self.close()
|
|
|
|
if res == "ok":
|
|
return True
|
|
else:
|
|
print repr(res)
|
|
return False
|
|
|
|
</t>
|
|
<t tx="aum.20040814002411">def __getitem__(self, item):
|
|
|
|
return self.get(item)
|
|
|
|
</t>
|
|
<t tx="aum.20040814002411.1">def __setitem__(self, item, val):
|
|
|
|
if not self.put(item, val):
|
|
raise Exception("Failed to insert")
|
|
|
|
</t>
|
|
<t tx="aum.20040814003559">def kill(self):
|
|
"""
|
|
Tells remote server to fall on its sword
|
|
"""
|
|
try:
|
|
while 1:
|
|
self.connect()
|
|
self.write("die\n")
|
|
self.flush()
|
|
self.close()
|
|
except:
|
|
pass
|
|
|
|
|
|
</t>
|
|
<t tx="aum.20040814004432">def finish(self):
|
|
|
|
SocketServer.StreamRequestHandler.finish(self)
|
|
|
|
</t>
|
|
<t tx="aum.20040814015747">def err(msg):
|
|
sys.stderr.write(msg+"\n")
|
|
</t>
|
|
<t tx="aum.20040814103533">def addref(self, ref):
|
|
"""
|
|
Passes a new noderef to node
|
|
"""
|
|
self.connect()
|
|
self.write("addref %s\n" % ref)
|
|
self.flush()
|
|
|
|
res = self.readline().strip()
|
|
|
|
self.close()
|
|
|
|
if res == "ok":
|
|
return True
|
|
else:
|
|
print repr(res)
|
|
return False
|
|
|
|
</t>
|
|
<t tx="aum.20040814110540">def userI2PDir(nodeName=None):
|
|
"""
|
|
Returns a directory under user's home dir into which
|
|
stasher files can be written
|
|
|
|
If nodename is given, a subdirectory will be found/created
|
|
|
|
Return value is toplevel storage dir if nodename not given,
|
|
otherwise absolute path including node dir
|
|
"""
|
|
if dataDir != None:
|
|
if not os.path.isdir(dataDir):
|
|
os.makedirs(dataDir)
|
|
return dataDir
|
|
|
|
if sys.platform == 'win32':
|
|
home = os.getenv("APPDATA")
|
|
if home:
|
|
topDir = os.path.join(home, "stasher")
|
|
else:
|
|
topDir = os.path.join(os.getcwd(), "stasher")
|
|
else:
|
|
#return os.path.dirname(__file__)
|
|
topDir = os.path.join(os.path.expanduser('~'), ".stasher")
|
|
|
|
if not os.path.isdir(topDir):
|
|
os.makedirs(topDir)
|
|
if nodeName == None:
|
|
return topDir
|
|
else:
|
|
nodeDir = os.path.join(topDir, nodeName)
|
|
if not os.path.isdir(nodeDir):
|
|
os.makedirs(nodeDir)
|
|
return nodeDir
|
|
|
|
</t>
|
|
<t tx="aum.20040814110755">def nodePidfile(nodename):
|
|
return os.path.join(userI2PDir(nodename), "node.pid")
|
|
|
|
</t>
|
|
<t tx="aum.20040814112703">def spawnproc(*args, **kw):
|
|
"""
|
|
Spawns a process and returns its PID
|
|
|
|
VOMIT!
|
|
|
|
I have to do a pile of odious for the win32 side
|
|
|
|
Returns a usable PID
|
|
|
|
Keywords:
|
|
- priority - priority at which to spawn - default 20 (highest)
|
|
"""
|
|
# get priority, convert to a unix 'nice' value
|
|
priority = 20 - kw.get('priority', 20)
|
|
|
|
if sys.platform != 'win32':
|
|
# *nix - easy
|
|
#print "spawnproc: launching %s" % repr(args)
|
|
|
|
# insert nice invocation
|
|
args = ['/usr/bin/nice', '-n', str(priority)] + list(args)
|
|
return os.spawnv(os.P_NOWAIT, args[0], args)
|
|
|
|
else:
|
|
# just close your eyes here and pretend this abomination isn't happening! :((
|
|
args = list(args)
|
|
args.insert(0, sys.executable)
|
|
cmd = " ".join(args)
|
|
#print "spawnproc: launching %s" % repr(cmd)
|
|
|
|
if 0:
|
|
try:
|
|
c = _winreg.ConnectRegistry(None, _winreg.HKEY_LOCAL_MACHINE)
|
|
c1 = _winreg.OpenKey(c, "SOFTWARE")
|
|
c2 = _winreg.OpenKey(c1, "Microsoft")
|
|
c3 = _winreg.OpenKey(c2, "Windows NT")
|
|
c4 = _winreg.OpenKey(c3, "CurrentVersion")
|
|
supportsBelowNormalPriority = 1
|
|
except:
|
|
supportsBelowNormalPriority = 0
|
|
else:
|
|
if sys.getwindowsversion()[3] != 2:
|
|
supportsBelowNormalPriority = 0
|
|
else:
|
|
supportsBelowNormalPriority = 1
|
|
|
|
# frig the priority into a windows value
|
|
if supportsBelowNormalPriority:
|
|
if priority < 7:
|
|
pri = win32process.IDLE_PRIORITY_CLASS
|
|
elif priority < 14:
|
|
pri = 0x4000
|
|
else:
|
|
pri = win32process.NORMAL_PRIORITY_CLASS
|
|
else:
|
|
if priority < 11:
|
|
pri = win32process.IDLE_PRIORITY_CLASS
|
|
else:
|
|
pri = win32process.NORMAL_PRIORITY_CLASS
|
|
|
|
print "spawnproc: launching %s" % repr(args)
|
|
si = win32process.STARTUPINFO()
|
|
hdl = win32process.CreateProcess(
|
|
sys.executable, # lpApplicationName
|
|
cmd, # lpCommandLine
|
|
None, # lpProcessAttributes
|
|
None, # lpThreadAttributes
|
|
0, # bInheritHandles
|
|
0, # dwCreationFlags
|
|
None, # lpEnvironment
|
|
None, # lpCurrentDirectory
|
|
si, # lpStartupInfo
|
|
)
|
|
pid = hdl[2]
|
|
#print "spawnproc: pid=%s" % pid
|
|
return pid
|
|
</t>
|
|
<t tx="aum.20040814112703.1">def killproc(pid):
|
|
if sys.platform == 'win32':
|
|
print repr(pid)
|
|
handle = win32api.OpenProcess(1, 0, pid)
|
|
print "pid %s -> %s" % (pid, repr(handle))
|
|
#return (0 != win32api.TerminateProcess(handle, 0))
|
|
win32process.TerminateProcess(handle, 0)
|
|
else:
|
|
return os.kill(pid, signal.SIGKILL)
|
|
</t>
|
|
<t tx="aum.20040814120624">def i2psocket(self, *args, **kw):
|
|
return i2p.socket.socket(*args, **kw)
|
|
|
|
</t>
|
|
<t tx="aum.20040814131117">def getref(self):
|
|
"""
|
|
Uplifts node's own ref
|
|
"""
|
|
self.connect()
|
|
self.write("getref\n")
|
|
self.flush()
|
|
|
|
res = self.readline().strip()
|
|
|
|
if res == "ok":
|
|
ref = self.readline().strip()
|
|
self.close()
|
|
return ref
|
|
else:
|
|
self.close()
|
|
return "failed"
|
|
|
|
</t>
|
|
<t tx="aum.20040814132559">@first #! /usr/bin/env python
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040814133042">@first #! /bin/sh
|
|
|
|
export I2PPYDIR=/main/i2p/cvs/i2p/apps/stasher/python
|
|
export WEBDIR=/main/i2p/services/stasher.i2p
|
|
export CVSDIR=/main/i2p/cvs/i2p/apps/stasher/python
|
|
|
|
cp README.txt $I2PPYDIR
|
|
cp stasher.py $I2PPYDIR/src
|
|
cp bencode.py $I2PPYDIR/src
|
|
cp code.leo $I2PPYDIR/src
|
|
cp *.stasher $I2PPYDIR/noderefs
|
|
cp *.stasher $WEBDIR/noderefs
|
|
|
|
cp stasher $I2PPYDIR/scripts
|
|
cp stasher-launch.py $I2PPYDIR/scripts/stasher.py
|
|
|
|
cp setup-stasher.py $I2PPYDIR/setup.py
|
|
|
|
# generate API dco
|
|
epydoc -n "Stasher Python API" -o api stasher.py
|
|
|
|
# make a release tarball
|
|
|
|
rm -rf release/*
|
|
export STVERSION=`./stasher.py --shortversion`
|
|
export TARDIR=release/stasher-$STVERSION
|
|
export DIRNAME=stasher-$STVERSION
|
|
export TARBALLNAME=stasher.tar.gz
|
|
mkdir $TARDIR
|
|
|
|
cp -a /main/i2p/cvs/i2p/apps/sam/python/i2p $TARDIR
|
|
cp -a code.leo stasher stasher.py bencode.py api $TARDIR
|
|
cp README-tarball.txt $TARDIR/README.txt
|
|
|
|
mkdir $TARDIR/noderefs
|
|
cp *.stasher $TARDIR/noderefs
|
|
|
|
cd release
|
|
|
|
tar cfz $TARBALLNAME $DIRNAME
|
|
|
|
# copy tarball and doco to websites
|
|
cp $TARBALLNAME $WEBDIR
|
|
cd ..
|
|
cp -a api $WEBDIR
|
|
|
|
# last but not least, commit to cvs
|
|
cp stasher.py $CVSDIR/src
|
|
cp *.stasher $CVSDIR/noderefs
|
|
cd $CVSDIR
|
|
cvs commit
|
|
</t>
|
|
<t tx="aum.20040814134235">@others
|
|
</t>
|
|
<t tx="aum.20040814140643">@first #! /usr/bin/env python
|
|
# wrapper script to run stasher node
|
|
|
|
# set this to the directory where you've installed stasher
|
|
stasherDir = "/path/to/my/stasher/dir"
|
|
|
|
import sys
|
|
sys.path.append(stasherDir)
|
|
import stasher
|
|
stasher.main()
|
|
</t>
|
|
<t tx="aum.20040814191506">@first #! /bin/sh
|
|
rm -rf /tmp/node1
|
|
stasher -V4 -Slocalhost:7656 -Clocalhost:7659 -d/tmp/node1 _start node1
|
|
|
|
</t>
|
|
<t tx="aum.20040814191718">@first #! /bin/sh
|
|
rm -rf /tmp/node2
|
|
stasher -V4 -Slocalhost:17656 -Clocalhost:17659 -d/tmp/node2 _start node2
|
|
|
|
</t>
|
|
<t tx="aum.20040814234015">@first #! /usr/bin/env python
|
|
"""
|
|
This is the installation script for Stasher, a distributed
|
|
file storage framework for I2P.
|
|
"""
|
|
|
|
import sys, os
|
|
from distutils.core import setup
|
|
|
|
oldcwd = os.getcwd()
|
|
os.chdir("src")
|
|
|
|
if sys.platform == 'win32':
|
|
stasherScript = "..\\scripts\\stasher.py"
|
|
else:
|
|
stasherScript = "../scripts/stasher"
|
|
|
|
|
|
try:
|
|
import i2p
|
|
import i2p.socket
|
|
import i2p.select
|
|
except:
|
|
print "Sorry, but you don't seem to have the core I2P"
|
|
print "python library modules installed."
|
|
print "If you're installing from cvs, please go to"
|
|
print "i2p/apps/sam/python, become root, and type:"
|
|
print " python setup.py install"
|
|
print "Then, retry this installation."
|
|
sys.exit(1)
|
|
|
|
setup(name="Stasher",
|
|
version="0.0",
|
|
description="Kademlia-based P2P distributed file storage app for I2P",
|
|
author="aum",
|
|
author_email="aum_i2p@hotmail.com",
|
|
url="http://stasher.i2p",
|
|
py_modules = ['stasher', 'bencode'],
|
|
scripts = [stasherScript],
|
|
)</t>
|
|
<t tx="aum.20040815000938"></t>
|
|
<t tx="aum.20040815000938.1"></t>
|
|
<t tx="aum.20040815000938.2"></t>
|
|
<t tx="aum.20040815001048">@nocolor
|
|
STASHER README
|
|
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040815143009">@nocolor
|
|
STASHER README
|
|
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040815143611">-----------------------
|
|
INSTALLING STASHER
|
|
|
|
Prerequisite:
|
|
|
|
Before you can install/run Stasher, you will first need to have installed
|
|
the I2P Python modules - available in cvs at i2p/apps/sam/python.
|
|
|
|
To install stasher, just make sure you've got the latest cvs, then type
|
|
python setup.py install
|
|
as root.
|
|
|
|
This installs the stasher engine, plus a wrapper client script called
|
|
'stasher', which setup.py will install into your execution path.
|
|
|
|
If you don't like the thought of becoming root, you could just put stasher.py
|
|
on your execution path, and/or create a symlink called 'stasher'.
|
|
|
|
Test your installation by typing 'stasher -h' - this should display
|
|
a help message.
|
|
|
|
</t>
|
|
<t tx="aum.20040815143611.1">------------------------
|
|
DOZE USERS PLEASE NOTE
|
|
|
|
You'll need to watch and see where the stasher.py
|
|
wrapper script gets installed. On my box, it ends up on d:\python23\scripts,
|
|
but on your box it'll likely go into c:\python23\scripts.
|
|
|
|
You may either update your system PATH environment variable to include your
|
|
python scripts directory, OR, you can copy stasher.py to anywhere that's
|
|
on your path.
|
|
|
|
In the explanations below, note that wherever I say to type 'stasher', you'll
|
|
need to type 'stasher.py' instead.
|
|
|
|
</t>
|
|
<t tx="aum.20040815143611.2">------------------------
|
|
WARNING
|
|
|
|
This is a very early pre-alpha test version of stasher.
|
|
It is only capable of storing or retrieving files of
|
|
less than 29k in size.
|
|
|
|
Also, files are totally insecure - anyone can overwrite any keys you
|
|
insert, and vice versa.
|
|
|
|
I'll be adding support for CHK-like and SSK-like keys in due course.
|
|
|
|
</t>
|
|
<t tx="aum.20040815143611.3">------------------------
|
|
USING STASHER
|
|
|
|
To see stasher's options, type:
|
|
|
|
stasher -h
|
|
|
|
This should dump out a verbose help message.
|
|
|
|
To start a stasher node, type:
|
|
|
|
stasher start
|
|
|
|
To shut down a stasher node, type:
|
|
|
|
stasher stop
|
|
|
|
To insert a file into stasher, type:
|
|
|
|
stasher put mykey myfile
|
|
|
|
Note, if you don't supply a filename, stasher can read
|
|
the file from standard input.
|
|
|
|
To retrieve a file from stasher, type:
|
|
|
|
stasher get mykey
|
|
|
|
</t>
|
|
<t tx="aum.20040815143611.4">INSTALLING STASHER FROM THE TARBALL
|
|
|
|
For regular users:
|
|
|
|
1. Crack this tarball, and copy the 'stasher-n.n.n' directory
|
|
somewhere safe, such as your home directory
|
|
|
|
2. Edit the small 'stasher' script therein, and set the variable
|
|
'stasherDir' as directed
|
|
|
|
3. Either put this directory onto your PATH, or create a symlink
|
|
called 'stasher' within any of your PATH dirs, pointing to
|
|
the stasher script
|
|
|
|
3. Test your installation by typing:
|
|
stasher -v
|
|
|
|
For windows users:
|
|
|
|
1. Make sure you have python2.3 or later installed
|
|
|
|
2. Untar this directory, and copy the directory into
|
|
C:\Program Files, or wherever you like to put your appz
|
|
|
|
3. Wherever you put the directory, add that to your system-wide
|
|
PATH environment variable
|
|
|
|
4. Test your installation by opening up an ugly black DOS window,
|
|
and typing:
|
|
stasher.py -v
|
|
|
|
5. Note - in the USAGE directions below, instead of typing 'stasher',
|
|
you'll need to type 'stasher.py'
|
|
|
|
</t>
|
|
<t tx="aum.20040815164410"></t>
|
|
<t tx="aum.20040815164410.1">class KRpcPingAll(KRpc):
|
|
"""
|
|
Pings all peers
|
|
"""
|
|
@others
|
|
</t>
|
|
<t tx="aum.20040815164410.2">type = 'pingall'
|
|
</t>
|
|
<t tx="aum.20040815164410.3">def __init__(self, localNode, client=None, **kw):
|
|
"""
|
|
Creates and launches a PINGALL rpc
|
|
|
|
Arguments:
|
|
- localNode - the node performing this RPC
|
|
- client - see KRpc.__init__
|
|
|
|
Keywords: none
|
|
"""
|
|
if kw.has_key('cbArgs'):
|
|
KRpc.__init__(self, localNode, client, cbArgs=kw['cbArgs'])
|
|
else:
|
|
KRpc.__init__(self, localNode, client)
|
|
|
|
</t>
|
|
<t tx="aum.20040815164410.4">def start(self):
|
|
"""
|
|
Kicks off this RPC
|
|
"""
|
|
# launch a findNode rpc against each of our peers
|
|
peers = self.localNode.peers
|
|
self.numSent = self.numPending = len(peers)
|
|
self.numReplied = self.numFailed = 0
|
|
for peer in peers:
|
|
KRpcPing(self.localNode, self.on_reply, peer=peer)
|
|
return
|
|
|
|
</t>
|
|
<t tx="aum.20040815164410.5">def returnValue(self, result):
|
|
"""
|
|
an override with a nicer call sig
|
|
"""
|
|
# a hack for testing - save this RPC object into the node
|
|
# so we can introspect it
|
|
self.localNode.lastrpc = self
|
|
|
|
try:
|
|
KRpc.returnValue(self, result, status=result)
|
|
except:
|
|
traceback.print_exc()
|
|
self.log(3, "Failed to return %s" % repr(result))
|
|
KRpc.returnValue(self, 0, status=0)
|
|
|
|
</t>
|
|
<t tx="aum.20040815164410.7">def on_reply(self, result):
|
|
"""
|
|
callback which fires when we get a reply from a STORE we sent to a
|
|
peer
|
|
"""
|
|
log(3, "got %s" % repr(result))
|
|
|
|
if result:
|
|
self.numReplied += 1
|
|
else:
|
|
self.numFailed += 1
|
|
self.numPending -= 1
|
|
|
|
if self.numPending <= 0:
|
|
res = "pinged:%s replied:%s timeout:%s" % (
|
|
self.numSent, self.numReplied, self.numFailed)
|
|
self.log(3, res)
|
|
self.returnValue(res)
|
|
|
|
</t>
|
|
<t tx="aum.20040815164410.8">def on_tick(self):
|
|
|
|
self.log(3, "this shouldn't have happened")
|
|
self.returnValue(False)
|
|
|
|
</t>
|
|
<t tx="aum.20040815170327">def _pingall(self, callback=None):
|
|
"""
|
|
Sends a ping to all peers, returns text string on replies/failures
|
|
"""
|
|
if callback:
|
|
KRpcPingAll(self, callback, **kw)
|
|
else:
|
|
return KRpcPingAll(self).execute()
|
|
|
|
|
|
</t>
|
|
<t tx="aum.20040815170456">def pingall(self):
|
|
"""
|
|
Uplifts node's own ref
|
|
"""
|
|
self.connect()
|
|
self.write("pingall\n")
|
|
self.flush()
|
|
|
|
res = self.readline().strip()
|
|
|
|
self.close()
|
|
|
|
return res
|
|
|
|
|
|
</t>
|
|
<t tx="aum.20040816014757">def returnValue(self, items):
|
|
"""
|
|
override with a nicer call sig
|
|
"""
|
|
# a hack for testing - save this RPC object into the node
|
|
# so we can introspect it
|
|
self.localNode.lastrpc = self
|
|
|
|
# another debugging hack
|
|
self.reportStats()
|
|
|
|
KRpc.returnValue(self, items, result=items)
|
|
|
|
</t>
|
|
<t tx="aum.20040816133040">def storeSplit(self):
|
|
"""
|
|
Gets called if we're splitting a big file into smaller chunks
|
|
|
|
Here, we:
|
|
- break the file up into chunks
|
|
- build a manifest
|
|
- launch store RPCs to store each chunk, where the key is SHA(chunk)
|
|
- launch a store RPC to store the 'manifest' (noting that if the manifest
|
|
is too big, it'll get recursively inserted as a splitfile as well
|
|
"""
|
|
# break up into chunks
|
|
chunks = []
|
|
hashes = []
|
|
size = len(self.value)
|
|
i = 0
|
|
self.nchunks = 0
|
|
while i < size:
|
|
chunks.append(self.value[i:i+maxValueSize])
|
|
hashes.append(shahash(chunks[-1]))
|
|
i += maxValueSize
|
|
self.nchunks += 1
|
|
|
|
# build the manifest
|
|
manifest = "chunks:%s\n%s\n" % (self.nchunks, "\n".join(hashes))
|
|
|
|
# set progress attributes
|
|
self.chunkManifestInserted = False
|
|
self.chunksInserted = 0
|
|
|
|
# launch nested Store RPCs for manifest, and each chunk
|
|
KRpcStore(self.localNode, self.on_doneChunkManifest,
|
|
local=self.isLocalOnly,
|
|
key=self.key,
|
|
value=manifest)
|
|
i = 0
|
|
while i < self.nchunks:
|
|
KRpcStore(self.localNode, self.on_doneChunk,
|
|
local=self.isLocalOnly,
|
|
key=hashes[i],
|
|
value=chunks[i])
|
|
i += 1
|
|
|
|
# now sit back and wait for the callbacks
|
|
</t>
|
|
<t tx="aum.20040816135222">def on_doneChunkManifest(self, result):
|
|
"""
|
|
Callback which fires when a manifest insert succeeds/fails
|
|
"""
|
|
# the chunk callback handles all
|
|
self.on_doneChunk(result, isManifest=True)
|
|
</t>
|
|
<t tx="aum.20040816135222.1">def on_doneChunk(self, result, isManifest=False):
|
|
"""
|
|
Callback which fires when a single chunk insert succeeds/fails
|
|
"""
|
|
# a failure either way means the whole RPC has failed
|
|
if not result:
|
|
# one huge fuck-up
|
|
self.returnValue(False)
|
|
return
|
|
|
|
# update our tally
|
|
if isManifest:
|
|
self.chunkManifestInserted = True
|
|
else:
|
|
self.chunksInserted += 1
|
|
|
|
# finished?
|
|
if self.chunkManifestInserted and (self.chunksInserted == self.nchunks):
|
|
# yep = success
|
|
self.returnValue(True)
|
|
|
|
</t>
|
|
<t tx="aum.20040816141128">def on_gotValue(self, value, hash=None):
|
|
"""
|
|
Callback which fires when we get the value stored under a key
|
|
|
|
Value is either the real value, or a splitfile manifest
|
|
If a real value, just return it.
|
|
If a splitfile manifest, launch nested findValue RPCs to get each chunk
|
|
"""
|
|
nchunks = 0
|
|
try:
|
|
firstline, rest = value.split("\n", 1)
|
|
firstline = firstline.strip()
|
|
kwd, str_nchunks = firstline.split(":")
|
|
if kwd != 'chunks':
|
|
raise hell
|
|
nchunks = int(nchunks)
|
|
value = rest
|
|
except:
|
|
pass # in this case, hell hath no fury at all
|
|
|
|
if nchunks == 0:
|
|
self.returnValue(value)
|
|
return
|
|
|
|
# now we get to the hard bit - we have to set up nested findData RPCs to
|
|
# get all the chunks and reassemble them
|
|
hashes = rest.strip().split("\n")
|
|
|
|
# do sanity checks
|
|
hashesAllValid = [len(h) == 40 for h in hashes]
|
|
if len(hashes) != nchunks:
|
|
self.log(
|
|
2,
|
|
"Splitfile retrieval failure\nmanifest contains %s hashes, should have been %s" % (
|
|
len(hashes), nchunks))
|
|
self.returnValue(None)
|
|
if False in hashesAllValid:
|
|
self.log(2, "Splitfile retrieval failure - one or more invalid hashes")
|
|
|
|
# now this is a bit weird - we need to bind each chunk to its hash, so we create a
|
|
# class which produces callables which fire our on_gotChunk callback
|
|
class ChunkNotifier:
|
|
def __init__(me, h, cb):
|
|
me.h = h
|
|
me.cb = cb
|
|
def __call__(me, val):
|
|
me.cb(me.h, val)
|
|
|
|
# now launch the chunk retrieval RPCs
|
|
# result is that for each retrieved chunk, our on_gotChunk callback will
|
|
# be invoked with the arguments (hash, value), so we can tick them off
|
|
self.numChunks = nchunks
|
|
self.numChunksReceived = 0
|
|
self.chunkHashes = hashes
|
|
self.chunks = dict.fromkeys(hashes)
|
|
for h in hashes:
|
|
KRpcFindData(self.localNode, h, ChunkNotifier(h, self.on_gotChunk))
|
|
|
|
# now, we can sit back and receive the chunks
|
|
|
|
</t>
|
|
<t tx="aum.20040816141128.1">def on_gotChunk(self, hexhash, value):
|
|
"""
|
|
Callback which fires when a nested chunk findNode returns
|
|
"""
|
|
if value == None:
|
|
self.log(2, "Chunk retrieval failed, fatal to this findData")
|
|
self.returnValue(None)
|
|
return
|
|
|
|
# got a value - vet it against hash
|
|
if shahash(value) != hexhash:
|
|
self.log(2, "Got a chunk, but it doesn't hash right - fatal to this findData")
|
|
self.returnValue(None)
|
|
return
|
|
|
|
# it's valid - stash it
|
|
self.chunks[hexhash] = value
|
|
self.numChunksReceived += 1
|
|
|
|
# have we finished yet?
|
|
if self.numChunksReceived <= self.numChunks:
|
|
# no
|
|
self.log(4, "Received chunk %s of %s" % (self.numChunksReceived, self.numChunks))
|
|
return
|
|
|
|
# maybe we have
|
|
self.log(4, "We appear to have all chunks, checking further")
|
|
|
|
# sanity check
|
|
if None in self.chunks.values():
|
|
self.log(2, "Fatal - reached chunk count, but chunks still missing")
|
|
self.returnValue(None)
|
|
return
|
|
|
|
# finally done - got all chunks, hashes are valid, reassemble in order
|
|
allChunks = [self.chunks[h] for h in self.chunkHashes]
|
|
reassembled = "".join(allChunks)
|
|
self.log(4, "Reassembled all %s chunks, SUCCESS" % self.numChunks)
|
|
self.returnValue(reassembled)
|
|
|
|
</t>
|
|
</tnodes>
|
|
</leo_file>
|