diff --git a/apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java b/apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java
index a1005f2e124b40802b58776f9b887b04c8b9ac1f..961e43b56fa1b941b9c70c87c2e00120e05a71b5 100644
--- a/apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java
+++ b/apps/routerconsole/java/src/net/i2p/router/web/LogsHelper.java
@@ -30,6 +30,7 @@ public class LogsHelper {
buf.append("<code>\n");
for (int i = msgs.size(); i > 0; i--) {
String msg = (String)msgs.get(i - 1);
+ msg = msg.replaceAll("<","<");
buf.append("<li>");
buf.append(msg);
buf.append("</li>\n");
@@ -46,6 +47,7 @@ public class LogsHelper {
buf.append("<code>\n");
for (int i = msgs.size(); i > 0; i--) {
String msg = (String)msgs.get(i - 1);
+ msg = msg.replaceAll("<","<");
buf.append("<li>");
buf.append(msg);
buf.append("</li>\n");
@@ -59,8 +61,10 @@ public class LogsHelper {
String str = FileUtil.readTextFile("wrapper.log", 500, false);
if (str == null)
return "";
- else
+ else {
+ str = str.replaceAll("<","<");
return "<pre>" + str + "</pre>";
+ }
}
public String getConnectionLogs() {
diff --git a/apps/syndie/java/src/net/i2p/syndie/BlogManager.java b/apps/syndie/java/src/net/i2p/syndie/BlogManager.java
index 786df04ae6a72cb0a9ea4cacf9394f4135868c17..9c235f987f25d99aad11cbaa8316ba341aa917bb 100644
--- a/apps/syndie/java/src/net/i2p/syndie/BlogManager.java
+++ b/apps/syndie/java/src/net/i2p/syndie/BlogManager.java
@@ -324,8 +324,10 @@ public class BlogManager {
}
public boolean isConfigured() {
- File cfg = getConfigFile();
- return (cfg.exists());
+ String p = _context.getProperty("syndie.configurationCheck");
+ if(p==null)
+ return false;
+ return true;
}
/**
@@ -348,6 +350,73 @@ public class BlogManager {
return delay;
}
+ public List getRssFeeds() {
+ List feedList = new ArrayList();
+ int i=0;
+ while(true) {
+ String url = _context.getProperty("syndie.rssFeed."+i+".url");
+ String blog = _context.getProperty("syndie.rssFeed."+i+".blog");
+ String tagPrefix = _context.getProperty("syndie.rssFeed."+i+".tagPrefix");
+ if(url==null || blog==null || tagPrefix==null)
+ break;
+ String feed[] = new String[3];
+ feed[0]=url.trim();
+ feed[1]=blog.trim();
+ feed[2]=tagPrefix.trim();
+ feedList.add(feed);
+ i++;
+ }
+ return feedList;
+ }
+ public boolean addRssFeed(String url, String blog, String tagPrefix) {
+
+ List feedList = getRssFeeds();
+ int nextIdx=feedList.size();
+
+ String baseFeedProp="syndie.rssFeed."+nextIdx;
+ System.setProperty(baseFeedProp+".url",url);
+ System.setProperty(baseFeedProp+".blog",blog);
+ System.setProperty(baseFeedProp+".tagPrefix",tagPrefix);
+ _log.info("addRssFeed("+nextIdx+"): "+url);
+ writeConfig();
+ Updater.wakeup();
+ return true;
+ }
+ public boolean deleteRssFeed(String url, String blog, String tagPrefix) {
+ List feedList = getRssFeeds();
+ Iterator iter = feedList.iterator();
+ int idx=0;
+ while(iter.hasNext()) {
+ String fields[] = (String[])iter.next();
+ if(fields[0].equals(url) &&
+ fields[1].equals(blog) &&
+ fields[2].equals(tagPrefix)) {
+ break;
+ }
+ idx++;
+ }
+
+ // copy any remaining to idx-1
+ while(iter.hasNext()) {
+ String fields[] = (String[])iter.next();
+ String baseFeedProp="syndie.rssFeed."+idx;
+ System.setProperty(baseFeedProp+".url",fields[0]);
+ System.setProperty(baseFeedProp+".blog",fields[1]);
+ System.setProperty(baseFeedProp+".tagPrefix",fields[2]);
+ idx++;
+ }
+
+ // Delete last idx from properties
+ String baseFeedProp="syndie.rssFeed."+idx;
+ System.getProperties().remove(baseFeedProp+".url");
+ System.getProperties().remove(baseFeedProp+".blog");
+ System.getProperties().remove(baseFeedProp+".tagPrefix");
+ _log.info("deleteRssFeed("+idx+"): "+url);
+ writeConfig();
+ return true;
+ }
+
+
public boolean authorizeAdmin(String pass) {
if (isSingleUser()) return true;
String admin = getAdminPasswordHash();
@@ -388,6 +457,7 @@ public class BlogManager {
if (defaultProxyPort > 0)
out.write("syndie.defaultProxyPort="+defaultProxyPort + "\n");
out.write("syndie.singleUser=" + isSingleUser + "\n");
+ out.write("syndie.configurationCheck=foo\n");
if (opts != null) {
for (Iterator iter = opts.keySet().iterator(); iter.hasNext(); ) {
String key = (String)iter.next();
@@ -525,10 +595,13 @@ public class BlogManager {
}
public BlogURI createBlogEntry(User user, String subject, String tags, String entryHeaders, String sml) {
- return createBlogEntry(user, subject, tags, entryHeaders, sml, null, null, null);
+ return createBlogEntry(user, true, subject, tags, entryHeaders, sml, null, null, null);
}
public BlogURI createBlogEntry(User user, String subject, String tags, String entryHeaders, String sml, List fileNames, List fileStreams, List fileTypes) {
- if (!user.getAuthenticated()) return null;
+ return createBlogEntry(user, true, subject, tags, entryHeaders, sml, fileNames, fileStreams, fileTypes);
+ }
+ public BlogURI createBlogEntry(User user, boolean shouldAuthenticate, String subject, String tags, String entryHeaders, String sml, List fileNames, List fileStreams, List fileTypes) {
+ if (shouldAuthenticate && !user.getAuthenticated()) return null;
BlogInfo info = getArchive().getBlogInfo(user.getBlog());
if (info == null) return null;
SigningPrivateKey privkey = getMyPrivateKey(info);
@@ -601,7 +674,10 @@ public class BlogManager {
if (ok) {
getArchive().regenerateIndex();
user.setMostRecentEntry(entryId);
- saveUser(user);
+ if(shouldAuthenticate)
+ saveUser(user);
+ else
+ storeUser(user);
return uri;
} else {
return null;
diff --git a/apps/syndie/java/src/net/i2p/syndie/Sucker.java b/apps/syndie/java/src/net/i2p/syndie/Sucker.java
index e84e767d81c05b38710300afb9ebab90b1a06bb2..75ebe66c1d083993bf76c9a3d633b229bcb5ceea 100644
--- a/apps/syndie/java/src/net/i2p/syndie/Sucker.java
+++ b/apps/syndie/java/src/net/i2p/syndie/Sucker.java
@@ -6,16 +6,14 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.net.MalformedURLException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
-import java.util.ListIterator;
-
-//import sun.security.provider.SHA;
+import java.util.Properties;
import com.sun.syndication.feed.synd.SyndCategory;
import com.sun.syndication.feed.synd.SyndContent;
@@ -27,9 +25,15 @@ import com.sun.syndication.io.XmlReader;
import net.i2p.I2PAppContext;
import net.i2p.data.Base64;
+import net.i2p.data.DataFormatException;
+import net.i2p.data.DataHelper;
+import net.i2p.data.Hash;
+import net.i2p.syndie.data.BlogURI;
import net.i2p.util.EepGet;
+import net.i2p.util.Log;
public class Sucker {
+ private static final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(Sucker.class);
private String urlToLoad;
private String outputDir="./sucker_out";
private String historyPath="./sucker.history";
@@ -46,9 +50,50 @@ public class Sucker {
private boolean pendingEndLink;
private boolean shouldProxy;
private int proxyPortNum;
+ private String blog;
+ private boolean pushToSyndie;
+ private long messageNumber=0;
+ private BlogManager bm;
+ private User user;
+
+ //
+ private List fileNames;
+ private List fileStreams;
+ private List fileTypes;
- public Sucker() {}
+ public Sucker() {
+ }
+ /**
+ * Constructor for BlogManager.
+ */
+ public Sucker(String[] strings) throws IllegalArgumentException {
+ pushToSyndie=true;
+ urlToLoad = strings[0];
+ blog = strings[1];
+ feedTag = strings[2];
+ outputDir = "blog-"+blog;
+ try {
+ historyPath=BlogManager.instance().getRootDir().getCanonicalPath()+"/rss.history";
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ proxyPort="4444";
+ proxyHost="localhost";
+
+ bm = BlogManager.instance();
+ Hash blogHash = new Hash();
+ try {
+ blogHash.fromBase64(blog);
+ } catch (DataFormatException e1) {
+ throw new IllegalArgumentException("ooh, bad $blog");
+ }
+
+ user = bm.getUser(blogHash);
+ if(user==null)
+ throw new IllegalArgumentException("wtf, user==null? hash:"+blogHash);
+ }
+
public boolean parseArgs(String args[]) {
for (int i = 0; i < args.length; i++) {
if ("--load".equals(args[i]))
@@ -77,58 +122,57 @@ public class Sucker {
if (urlToLoad == null)
return false;
-
- // Find base url, gah HELP
- int idx=urlToLoad.length();
- int x=urlToLoad.indexOf('?');
- if(x>0)
- idx=x;
- while(idx>0)
- {
- idx--;
- if(urlToLoad.charAt(idx)=='/')
- break;
- }
- if(idx==0)
- idx=x;
- baseUrl=urlToLoad.substring(0,idx);
-
- System.out.println("Processing: "+urlToLoad);
return true;
}
+ /**
+ * Fetch urlToLoad and call convertToHtml() on any new entries.
+ */
public void suck() {
SyndFeed feed;
+
+ // Find base url
+ int idx=urlToLoad.lastIndexOf('/');
+ if(idx>0)
+ baseUrl=urlToLoad.substring(0,idx);
+ else
+ baseUrl=urlToLoad;
+
+ debugLog("Processing: "+urlToLoad);
+ debugLog("Base url: "+baseUrl);
//
try {
-
- // Create outputDir if missing
- File f = new File(outputDir);
- f.mkdirs();
+ File lastIdFile=null;
+
+ // Get next message number to use (for messageId in history only)
+ if(!pushToSyndie) {
+
+ lastIdFile = new File(historyPath + ".lastId");
+ if (!lastIdFile.exists())
+ lastIdFile.createNewFile();
+
+ FileInputStream fis = new FileInputStream(lastIdFile);
+ String number = readLine(fis);
+ try {
+ messageNumber = Integer.parseInt(number);
+ } catch (NumberFormatException e) {
+ messageNumber = 0;
+ }
+
+ // Create outputDir if missing
+ File f = new File(outputDir);
+ f.mkdirs();
+ } else {
+ messageNumber=bm.getNextBlogEntry(user);
+ }
// Create historyFile if missing
historyFile = new File(historyPath);
if (!historyFile.exists())
historyFile.createNewFile();
- int messageNumber;
-
- File lastIdFile = new File(historyPath + ".lastId");
- if (!lastIdFile.exists())
- lastIdFile.createNewFile();
-
- FileInputStream fis = new FileInputStream(lastIdFile);
- String number = readLine(fis);
- try {
- messageNumber = Integer.parseInt(number);
- } catch (NumberFormatException e) {
- messageNumber = 0;
- }
-
- SyndFeedInput input = new SyndFeedInput();
-
shouldProxy = false;
proxyPortNum = -1;
if ( (proxyHost != null) && (proxyPort != null) ) {
@@ -140,11 +184,11 @@ public class Sucker {
nfe.printStackTrace();
}
}
+
+ // fetch
int numRetries = 2;
- // perhaps specify a temp dir?
File fetched = File.createTempFile("sucker", ".fetch");
fetched.deleteOnExit();
- // we use eepGet, since it retries and doesn't leak DNS requests like URL does
EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum,
numRetries, fetched.getAbsolutePath(), urlToLoad);
SuckerFetchListener lsnr = new SuckerFetchListener();
@@ -155,89 +199,71 @@ public class Sucker {
System.err.println("Unable to retrieve the url after " + numRetries + " tries.");
return;
}
-
+ if(get.getNotModified()) {
+ debugLog("not modified, saving network bytes from useless fetch");
+ return;
+ }
+
+ // Build entry list from fetched rss file
+ SyndFeedInput input = new SyndFeedInput();
feed = input.build(new XmlReader(fetched));
List entries = feed.getEntries();
FileOutputStream hos = new FileOutputStream(historyFile, true);
- ListIterator iter = entries.listIterator();
- while (iter.hasNext()) {
+ // Process list backwards to get syndie to display the
+ // entries in the right order. (most recent at top)
+ for (int i = entries.size()-1; i >= 0; i--) {
+ SyndEntry e = (SyndEntry) entries.get(i);
attachmentCounter=0;
- SyndEntry e = (SyndEntry) iter.next();
- // Calculate messageId
- String feedHash = sha1(urlToLoad);
- String itemHash = sha1(e.getTitle() + e.getDescription());
- Date d = e.getPublishedDate();
- String time;
- if(d!=null)
- time = "" + d.getTime();
- else
- time = "" + new Date().getTime();
-
- String outputFileName = outputDir + "/" + messageNumber;
-
- /*
- * $feedHash:$itemHash:$time:$outputfile. $feedHash would be the
- * hash (md5? sha1? sha2?) of the $urlToFeed, $itemHash is some
- * hash of the SyndEntry, $time would be the time that the entry
- * was posted $outputfile would be the $outputdir/$uniqueid
- */
- String messageId = feedHash + ":" + itemHash + ":" + time + ":"
- + outputFileName;
-
- // Check if we already have this
- if (!existsInHistory(messageId)) {
- System.out.println("new: " + messageId);
-
- if (convertToSml(e, ""+messageNumber)) {
-
- if (pushScript != null) {
- if (!execPushScript(""+messageNumber, time)) {
- System.out.println("################## push failed");
- }else {
- System.out.println("push success");
- hos.write(messageId.getBytes());
- hos.write("\n".getBytes());
- }
- }
- }
- messageNumber++;
+ String messageId = convertToSml(e);
+ if (messageId!=null) {
+ hos.write(messageId.getBytes());
+ hos.write("\n".getBytes());
}
}
-
- FileOutputStream fos = new FileOutputStream(lastIdFile);
- fos.write(("" + messageNumber).getBytes());
+
+ if(!pushToSyndie) {
+ FileOutputStream fos = new FileOutputStream(lastIdFile);
+ fos.write(("" + messageNumber).getBytes());
+ }
} catch (MalformedURLException e) {
- // TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalArgumentException e) {
- // TODO Auto-generated catch block
e.printStackTrace();
} catch (FeedException e) {
- // TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
- // TODO Auto-generated catch block
e.printStackTrace();
}
- System.out.println("Done.");
+ debugLog("Done.");
}
public static void main(String[] args) {
Sucker sucker = new Sucker();
boolean ok = sucker.parseArgs(args);
if (!ok) {
- usage();
- return;
+ System.out.println("sucker --load $urlToFeed \n"
+ + "--proxyhost <host> \n"
+ + "--proxyport <port> \n"
+ + "--importenclosures true \n"
+ + "--importrefs true \n"
+ + "--tag feed \n"
+ + "--outputdir ./sucker_out \n"
+ + "--exec pushscript.sh OUTPUTDIR UNIQUEID ENTRYTIMESTAMP \n"
+ + "--history ./sucker.history");
+ System.exit(1);
}
sucker.suck();
}
+ /**
+ * Call the specified script with "$outputDir $id and $time".
+ */
private boolean execPushScript(String id, String time) {
try {
String ls_str;
@@ -251,7 +277,7 @@ public class Sucker {
try {
while ((ls_str = ls_in.readLine()) != null) {
- System.out.println(pushScript + ": " + ls_str);
+ debugLog(pushScript + ": " + ls_str);
}
} catch (IOException e) {
return false;
@@ -261,7 +287,6 @@ public class Sucker {
if(pushScript_proc.exitValue()==0)
return true;
} catch (InterruptedException e) {
- // TODO Auto-generated catch block
e.printStackTrace();
}
return false;
@@ -271,40 +296,57 @@ public class Sucker {
}
}
- private boolean convertToSml(SyndEntry e, String messageName) {
-
- // Create message
- FileOutputStream fos;
- messagePath=outputDir+"/"+messageName;
+ /**
+ * Converts the SyndEntry e to sml and fetches any images as attachments
+ */
+ private String convertToSml(SyndEntry e) {
+ String subject;
+
+ // Calculate messageId, and check if we have got the message already
+ String feedHash = sha1(urlToLoad);
+ String itemHash = sha1(e.getTitle() + e.getDescription());
+ Date d = e.getPublishedDate();
+ String time;
+ if(d!=null)
+ time = "" + d.getTime();
+ else
+ time = "" + new Date().getTime();
+ String outputFileName = outputDir + "/" + messageNumber;
+ String messageId = feedHash + ":" + itemHash + ":" + time + ":" + outputFileName;
+ // Check if we already have this
+ if (existsInHistory(messageId))
+ return null;
+
+ debugLog("new: " + messageId);
+
try {
- fos = new FileOutputStream(messagePath);
- String sml;
- sml = "Subject: " + e.getTitle() + "\n";
+ String sml="";
+ subject=e.getTitle();
List cats = e.getCategories();
Iterator iter = cats.iterator();
String tags = feedTag;
while (iter.hasNext()) {
SyndCategory c = (SyndCategory) iter.next();
+ debugLog("Name: "+c.getName());
+ debugLog("uri:"+c.getTaxonomyUri());
String tag=c.getName();
tag=tag.replaceAll("[^a-zA-z.-_:]","_");
tags += "\t" + feedTag + "." + tag;
}
- sml += "Tags: " + tags + "\n";
- sml += "\n";
SyndContent content;
List l = e.getContents();
if(l!=null)
{
- System.out.println("There is content");
+ debugLog("There is content");
iter = l.iterator();
while(iter.hasNext())
{
content = (SyndContent)iter.next();
String c = content.getValue();
- System.out.println("Content: "+c);
+ debugLog("Content: "+c);
sml += htmlToSml(c);
sml += "\n";
}
@@ -314,17 +356,53 @@ public class Sucker {
source=baseUrl+source;
sml += "[link schema=\"web\" location=\""+source+"\"]source[/link]\n";
- fos.write(sml.getBytes());
-
- return true;
+ if(pushToSyndie) {
+ debugLog("user.blog: "+user.getBlogStr());
+ debugLog("user.id: "+bm.getNextBlogEntry(user));
+ debugLog("subject: "+subject);
+ debugLog("tags: "+tags);
+ debugLog("sml: "+sml);
+ debugLog("");
+ BlogURI uri = bm.createBlogEntry(
+ user,
+ false,
+ subject,
+ tags,
+ null,
+ sml,
+ fileNames,
+ fileStreams,
+ fileTypes);
+
+ if(uri==null) {
+ debugLog("pushToSyndie failure.");
+ return null;
+ }
+ else
+ debugLog("pushToSyndie success, uri: "+uri.toString());
+ }
+ else
+ {
+ FileOutputStream fos;
+ fos = new FileOutputStream(messagePath);
+ sml=subject + "\nTags: " + tags + "\n\n" + sml;
+ fos.write(sml.getBytes());
+ if (pushScript != null) {
+ if (!execPushScript(""+messageNumber, time)) {
+ debugLog("################## push failed");
+ } else {
+ debugLog("push success");
+ }
+ }
+ }
+ messageNumber++;
+ return messageId;
} catch (FileNotFoundException e1) {
- // TODO Auto-generated catch block
e1.printStackTrace();
} catch (IOException e2) {
- // TODO Auto-generated catch block
e2.printStackTrace();
}
- return false;
+ return null;
}
private String htmlToSml(String html) {
@@ -338,29 +416,32 @@ public class Sucker {
{
if(html.charAt(i)=='<')
{
- //System.out.println("html: "+html.substring(i));
+ //log("html: "+html.substring(i));
int tagLen = findTagLen(html.substring(i));
if(tagLen<=0)
{
- System.out.println("Bad html? ("+html+")");
+ debugLog("Bad html? ("+html+")");
break;
}
//
String htmlTag = html.substring(i,i+tagLen);
- //System.out.println("htmlTag: "+htmlTag);
+ //log("htmlTag: "+htmlTag);
String smlTag = htmlTagToSmlTag(htmlTag);
if(smlTag!=null)
sml+=smlTag;
i+=tagLen;
- //System.out.println("tagLen: "+tagLen);
+ //log("tagLen: "+tagLen);
sml+=" ";
}
else
{
- sml+=html.charAt(i++);
+ char c=html.charAt(i++);
+ sml+=c;
+ if(c=='[' || c==']')
+ sml+=c;
}
}
@@ -373,7 +454,7 @@ public class Sucker {
if(importEnclosures && htmlTagLowerCase.startsWith("<img"))
{
- System.out.println("Found image tag: "+htmlTag);
+ debugLog("Found image tag: "+htmlTag);
int a,b;
a=htmlTagLowerCase.indexOf("src=\"")+5;
b=a+1;
@@ -383,7 +464,7 @@ public class Sucker {
if(pendingEndLink) {
ret="[/link]";
- pendingEndLink=false;
+ pendingEndLink=false;
}
ret += "[img attachment=\""+""+ attachmentCounter +"\"]";
@@ -391,11 +472,13 @@ public class Sucker {
a=htmlTagLowerCase.indexOf("alt=\"")+5;
if(a>=5)
{
- b=a+1;
- while(htmlTagLowerCase.charAt(b)!='\"')
- b++;
- String altText=htmlTag.substring(a,b);
- ret+=altText;
+ b=a;
+ if(htmlTagLowerCase.charAt(b)!='\"') {
+ while(htmlTagLowerCase.charAt(b)!='\"')
+ b++;
+ String altText=htmlTag.substring(a,b);
+ ret+=altText;
+ }
}
ret+="[/img]";
@@ -405,14 +488,14 @@ public class Sucker {
fetchAttachment(imageLink);
- System.out.println("Converted to: "+ret);
+ debugLog("Converted to: "+ret);
return ret;
}
if(importRefs && htmlTagLowerCase.startsWith("<a "))
{
- System.out.println("Found link tag: "+htmlTag);
+ debugLog("Found link tag: "+htmlTag);
int a,b;
a=htmlTagLowerCase.indexOf("href=\"")+6;
@@ -426,9 +509,12 @@ public class Sucker {
String schema="web";
ret += "[link schema=\""+schema+"\" location=\""+link+"\"]";
- pendingEndLink=true;
+ if(htmlTagLowerCase.endsWith("/>"))
+ ret += "[/link]";
+ else
+ pendingEndLink=true;
- System.out.println("Converted to: "+ret);
+ debugLog("Converted to: "+ret);
return ret;
}
@@ -446,19 +532,36 @@ public class Sucker {
return "[i]";
if("</i>".equals(htmlTagLowerCase))
return "[/i]";
+ if("<em>".equals(htmlTagLowerCase))
+ return "[i]";
+ if("</em>".equals(htmlTagLowerCase))
+ return "[/i]";
return null;
}
private void fetchAttachment(String link) {
- System.out.println("Fetch attachment from: "+link);
+ link=link.replaceAll("&","&");
- String attachmentPath = messagePath+"."+attachmentCounter;
- link=link.replaceAll("&","&");
-
+ debugLog("Fetch attachment from: "+link);
+
+ File fetched;
+ if(pushToSyndie) {
+ try {
+ // perhaps specify a temp dir?
+ fetched = File.createTempFile("sucker",".attachment");
+ fetched.deleteOnExit();
+ } catch (IOException e) {
+ e.printStackTrace();
+ return;
+ }
+ fetched.deleteOnExit();
+ } else {
+ String attachmentPath = messagePath+"."+attachmentCounter;
+ fetched = new File(attachmentPath);
+ }
int numRetries = 2;
- File fetched = new File(attachmentPath);
// we use eepGet, since it retries and doesn't leak DNS requests like URL does
EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum,
numRetries, fetched.getAbsolutePath(), link);
@@ -471,9 +574,31 @@ public class Sucker {
fetched.delete();
return;
}
+ String filename=EepGet.suggestName(link);
+ String contentType = get.getContentType();
+ if(contentType==null)
+ contentType="text/plain";
+ debugLog("successful fetch of filename "+filename);
+ if(fileNames==null) fileNames = new ArrayList();
+ if(fileTypes==null) fileTypes = new ArrayList();
+ if(fileStreams==null) fileStreams = new ArrayList();
+ fileNames.add(filename);
+ fileTypes.add(contentType);
+ try {
+ fileStreams.add(new FileInputStream(fetched));
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
attachmentCounter++;
}
+ private void debugLog(String string) {
+ if (_log.shouldLog(Log.DEBUG))
+ _log.debug(string);
+ if(!pushToSyndie)
+ System.out.println(string);
+ }
+
private static int findTagLen(String s) {
int i;
for(i=0;i<s.length();i++)
@@ -487,7 +612,7 @@ public class Sucker {
i++;
}
}
- System.out.println("WTF");
+ System.out.println("WTF in Sucker.findTagLen("+s+")");
return -1;
}
@@ -513,25 +638,11 @@ public class Sucker {
return true;
}
} catch (FileNotFoundException e) {
- // TODO Auto-generated catch block
e.printStackTrace();
}
return false;
}
- private static void usage() {
- System.out.println("sucker --load $urlToFeed \n"
- + "--proxyhost <host> \n"
- + "--proxyport <port> \n"
- + "--importenclosures true \n"
- + "--importrefs true \n"
- + "--tag feed \n"
- + "--outputdir ./sucker_out \n"
- + "--exec pushscript.sh OUTPUTDIR UNIQUEID ENTRYTIMESTAMP \n"
- + "--history ./sucker.history");
- System.exit(1);
- }
-
private static String sha1(String s) {
try {
MessageDigest md = MessageDigest.getInstance("SHA");
@@ -552,7 +663,6 @@ public class Sucker {
try {
c = in.read();
} catch (IOException e) {
- // TODO Auto-generated catch block
e.printStackTrace();
break;
}
@@ -562,7 +672,6 @@ public class Sucker {
break;
sb.append((char) c);
}
- // TODO Auto-generated method stub
return sb.toString();
}
}
diff --git a/apps/syndie/java/src/net/i2p/syndie/Updater.java b/apps/syndie/java/src/net/i2p/syndie/Updater.java
index 6e54dfe221bf71be163cbeb99ccbcad8964de790..5f16f0a80d303ca600cd1e729a2459bd793eaf07 100644
--- a/apps/syndie/java/src/net/i2p/syndie/Updater.java
+++ b/apps/syndie/java/src/net/i2p/syndie/Updater.java
@@ -1,6 +1,8 @@
package net.i2p.syndie;
import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
import net.i2p.I2PAppContext;
import net.i2p.util.Log;
@@ -23,6 +25,12 @@ public class Updater {
for (int i = 0; i < archives.length; i++) {
fetchArchive(archives[i]);
}
+ List rssFeeds = bm.getRssFeeds();
+ Iterator iter = rssFeeds.iterator();
+ while(iter.hasNext()) {
+ Sucker sucker = new Sucker((String[])iter.next());
+ sucker.suck();
+ }
}
public void fetchArchive(String archive) {
diff --git a/apps/syndie/jsp/_topnav.jsp b/apps/syndie/jsp/_topnav.jsp
index 17fcd77b017ea507f2afb08ad461c6c6176038c2..f1850c3022a7c72496037f488e2fe8ad7e8097b2 100644
--- a/apps/syndie/jsp/_topnav.jsp
+++ b/apps/syndie/jsp/_topnav.jsp
@@ -5,6 +5,7 @@
<span class="b_topnavHome"><a href="index.jsp" class="b_topnavHome">Home</a></span>
<a href="admin.jsp" class="b_topnavAdmin">Syndie admin</a>
<a href="remote.jsp" class="b_topnavRemote">Remote archives</a>
+<a href="rssimport.jsp" class="b_topnavRSSImport">RSS imports</a>
<a href="import.jsp" class="b_topnavImport">Import</a>
</td><td nowrap="nowrap" height="10" class="b_topnavUser"><%
if ("true".equals(request.getParameter("logout"))) {
diff --git a/apps/syndie/jsp/rssimport.jsp b/apps/syndie/jsp/rssimport.jsp
new file mode 100644
index 0000000000000000000000000000000000000000..4ac999fc204e9e0444ae4ca692182b00e14474f4
--- /dev/null
+++ b/apps/syndie/jsp/rssimport.jsp
@@ -0,0 +1,143 @@
+<%@page contentType="text/html; charset=UTF-8" pageEncoding="UTF-8" import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.data.*, net.i2p.syndie.*, org.mortbay.servlet.MultiPartRequest, java.util.*, java.io.*" %><%
+request.setCharacterEncoding("UTF-8"); %><jsp:useBean scope="session" class="net.i2p.syndie.User" id="user"
+/><!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 TRANSITIONAL//EN" "http://www.w3c.org/TR/1999/REC-html401-19991224/loose.dtd">
+<html>
+<head>
+<title>SyndieMedia rss import configuration</title>
+<link href="style.jsp" rel="stylesheet" type="text/css" >
+</head>
+<body>
+<table border="1" cellpadding="0" cellspacing="0" width="100%">
+<tr class="b_toplogo"><td colspan="5" valign="top" align="left" class="b_toplogo"><jsp:include page="_toplogo.jsp" /></td></tr>
+<tr><td valign="top" align="left" rowspan="2" class="b_leftnav"><jsp:include page="_leftnav.jsp" /></td>
+ <jsp:include page="_topnav.jsp" />
+ <td valign="top" align="left" rowspan="2" class="b_rightnav"><jsp:include page="_rightnav.jsp" /></td></tr>
+<tr class="b_content"><td valign="top" align="left" colspan="3" class="b_content"><%
+
+BlogManager bm = BlogManager.instance();
+if (!user.getAuthenticated()) {
+ %><span class="b_rssMsgErr">Please log in.</span><%
+}
+else if(!bm.authorizeRemote(user)) {
+ %><span class="b_rssMsgErr">You are not authorized for remote access.</span><%
+} else {
+ String url=request.getParameter("url");
+ if(url!=null) url=url.trim();
+ String blog=request.getParameter("blog");
+ if(blog!=null) blog=blog.trim();
+ String tagPrefix=request.getParameter("tagprefix");
+ if(tagPrefix!=null) tagPrefix=tagPrefix.trim();
+ String action = request.getParameter("action");
+ if ( (action != null) && ("Add".equals(action)) ) {
+ if(url==null || blog==null || tagPrefix==null) {
+ %><span class="b_rssImportMsgErr">Please fill in all fields</span><br /><%
+ } else {
+ boolean ret=bm.addRssFeed(url,blog,tagPrefix);
+ if(!ret) {
+ %><span class="b_rssImportMsgErr">addRssFeed failure.</span><%
+ } else {
+ %><span class="b_rssImportMsgOk">RSS feed added.</span><%
+ }
+ }
+ } else if ( (action != null) && ("Change".equals(action)) ) {
+ String lastUrl=request.getParameter("lasturl");
+ String lastBlog=request.getParameter("lastblog");
+ String lastTagPrefix=request.getParameter("lasttagprefix");
+ if(url==null || blog==null || tagPrefix==null || lastUrl==null || lastBlog==null || lastTagPrefix==null) {
+ %><span class="b_rssImportMsgErr">error, some fields were empty.</span><br /><%
+ } else {
+ boolean ret=bm.deleteRssFeed(lastUrl,lastBlog,lastTagPrefix);
+ if(!ret) {
+ %><span class="b_rssImportMsgErr">Could not delete while attempting to change.</span><%
+ } else {
+ ret=bm.addRssFeed(url,blog,tagPrefix);
+ if(!ret) {
+ %><span class="b_rssImportMsgErr">Could not add while attempting to change.</span><%
+ } else {
+ %><span class="b_rssImportMsgOk">Ok, changed successfully.</span><%
+ }
+ }
+ }
+ } else if ( (action != null) && ("Delete".equals(action)) ) {
+ %><span class="b_rssImportMsgErr">Delete some thing</span><br /><%
+ if(url==null || blog==null || tagPrefix==null) {
+ %><span class="b_rssImportMsgErr">error, some fields were empty.</span><br /><%
+ } else {
+ boolean ret=bm.instance().deleteRssFeed(url,blog,tagPrefix);
+ if(!ret) {
+ %><span class="b_rssImportMsgErr">error, could not delete.</span><%
+ } else {
+ %><span class="b_rssImportMsgOk">ok, deleted successfully.</span><%
+ }
+ }
+ }
+ String blogStr=user.getBlogStr();
+ if(blogStr==null)
+ blogStr="";
+
+ ///////////////////////////////////////////////
+%>
+<p>Here you can add RSS feeds that will be periodically polled and added to your syndie. </p>
+<form action="rssimport.jsp" method="POST">
+RSS URL. (e.g. http://tracker.postman.i2p/rss.php)<br />
+<em><span class="b_rssImportField">url:</span></em> <input class="b_rssImportField" type="text" size="50" name="url" /><br />
+Blog hash to which the RSS entries will get posted, defaults to the one you're logged in to.<br />
+<em><span class="b_rssImportField">blog:</span></em> <input class="b_rssImportField" type="text" <%="value=\""+blogStr+"\""%> size="20" name="blog" /><br />
+This will be prepended to any tags that the RSS feed contains. (e.g. feed.tracker)<br />
+<em><span class="b_rssImportField">tagprefix:</span></em> <input class="b_rssImportField" type="text" value="feed" size="20" name="tagprefix" /><br />
+<input class="b_rssImportSubmit" type="submit" name="action" value="Add" /> <input class="b_rssImportCancel" type="reset" value="Cancel" /></form>
+<%
+///////////////////////////////////////////////
+ List feedList = bm.getRssFeeds();
+ if(feedList.size()>0) {
+%>
+<hr /><h3>Subscriptions:</h3><br />
+<table border="0" width="100%" class="b_rss">
+<tr class="b_rssHeader">
+<td class="b_rssHeader"><em class="b_rssHeader">Url</em></td>
+<td class="b_rssHeader"><em class="b_rssHeader">Blog</em></td>
+<td class="b_rssHeader"><em class="b_rssHeader">TagPrefix</em></td>
+<td class="b_rssHeader"> </td></tr>
+<%
+ Iterator iter = feedList.iterator();
+ while(iter.hasNext()) {
+ String fields[]=(String[])iter.next();
+ url=fields[0];
+ blog=fields[1];
+ tagPrefix=fields[2];
+ StringBuffer buf = new StringBuffer(128);
+
+ buf.append("<tr class=\"b_rssDetail\"><form action=\"rssimport.jsp\" method=\"POST\">");
+ buf.append("<input type=\"hidden\" name=\"lasturl\" value=\"").append(url).append("\" />");
+ buf.append("<input type=\"hidden\" name=\"lastblog\" value=\"").append(blog).append("\" />");
+ buf.append("<input type=\"hidden\" name=\"lasttagprefix\" value=\"").append(tagPrefix).append("\" />");
+
+ buf.append("<td class=\"b_rssUrl\"><input class=\"b_rssUrl\" type=\"text\" size=\"50\" name=\"url\" value=\"").append(url).append("\" /></td>");
+ buf.append("<td class=\"b_rssBlog\"><input class=\"b_rssBlog\" type=\"text\" size=\"20\" name=\"blog\" value=\"").append(blog).append("\" /></td>");
+ buf.append("<td class=\"b_rssPrefix\"><input class=\"b_rssPrefix\" type=\"text\" size=\"20\" name=\"tagprefix\" value=\"").append(tagPrefix).append("\" /></td>");
+ buf.append("<td class=\"b_rssDetail\" nowrap=\"nowrap\">");
+
+ buf.append("<input class=\"b_rssChange\" type=\"submit\" name=\"action\" value=\"Change\" />");
+ buf.append("<input class=\"b_rssDelete\" type=\"submit\" name=\"action\" value=\"Delete\" />");
+
+ buf.append("</td></form></tr>");
+ out.write(buf.toString());
+ buf.setLength(0);
+ }
+ }
+/*
+<p><h3>todo:</h3>
+<p>caching (eepget should do it)
+<p>enclosures support (requires cvs rome)
+<p>syndie.sucker.minHistory/maxHistory used to roll over the history file?
+<p>configurable update period
+*/
+%>
+</table>
+<hr />
+<%
+}
+%>
+</td></tr>
+</table>
+</body>
diff --git a/core/java/src/net/i2p/util/EepGet.java b/core/java/src/net/i2p/util/EepGet.java
index a52ae0d87365f577702c234c26ce2c68463328b0..e3dde46b8e7040ea30490c056d10b475568279df 100644
--- a/core/java/src/net/i2p/util/EepGet.java
+++ b/core/java/src/net/i2p/util/EepGet.java
@@ -51,6 +51,7 @@ public class EepGet {
private String _etag;
private boolean _encodingChunked;
private boolean _notModified;
+ private String _contentType;
public EepGet(I2PAppContext ctx, String proxyHost, int proxyPort, int numRetries, String outputFile, String url) {
this(ctx, true, proxyHost, proxyPort, numRetries, outputFile, url);
@@ -209,11 +210,13 @@ public class EepGet {
if (timeToSend > 0) {
StringBuffer buf = new StringBuffer(50);
buf.append(" ");
- double pct = ((double)alreadyTransferred + (double)_written) / ((double)alreadyTransferred + (double)bytesRemaining);
- synchronized (_pct) {
- buf.append(_pct.format(pct));
+ if ( bytesRemaining > 0 ) {
+ double pct = ((double)alreadyTransferred + (double)_written) / ((double)alreadyTransferred + (double)bytesRemaining);
+ synchronized (_pct) {
+ buf.append(_pct.format(pct));
+ }
+ buf.append(": ");
}
- buf.append(": ");
buf.append(_written+alreadyTransferred);
buf.append(" @ ");
double lineKBytes = ((double)_markSize * (double)_lineSize)/1024.0d;
@@ -243,9 +246,15 @@ public class EepGet {
if (notModified) {
System.out.println("== Source not modified since last download");
} else {
- System.out.println("== Transfer of " + url + " completed with " + (alreadyTransferred+bytesTransferred)
- + " and " + (bytesRemaining - bytesTransferred) + " remaining");
- System.out.println("== Output saved to " + outputFile);
+ if ( bytesRemaining > 0 ) {
+ System.out.println("== Transfer of " + url + " completed with " + (alreadyTransferred+bytesTransferred)
+ + " and " + (bytesRemaining - bytesTransferred) + " remaining");
+ System.out.println("== Output saved to " + outputFile);
+ } else {
+ System.out.println("== Transfer of " + url + " completed with " + (alreadyTransferred+bytesTransferred)
+ + " bytes transferred");
+ System.out.println("== Output saved to " + outputFile);
+ }
}
long timeToSend = _context.clock().now() - _startedOn;
System.out.println("== Transfer time: " + DataHelper.formatDuration(timeToSend));
@@ -355,9 +364,19 @@ public class EepGet {
_out.write(buf, 0, read);
_bytesTransferred += read;
remaining -= read;
+ if (remaining==0 && _encodingChunked) {
+ if(_proxyIn.read()=='\r' && _proxyIn.read()=='\n') {
+ remaining = (int) readChunkLength();
+ }
+ }
if (read > 0)
for (int i = 0; i < _listeners.size(); i++)
- ((StatusListener)_listeners.get(i)).bytesTransferred(_alreadyTransferred, read, _bytesTransferred, _bytesRemaining, _url);
+ ((StatusListener)_listeners.get(i)).bytesTransferred(
+ _alreadyTransferred,
+ read,
+ _bytesTransferred,
+ _encodingChunked?-1:_bytesRemaining,
+ _url);
}
if (_out != null)
@@ -369,7 +388,13 @@ public class EepGet {
if ( (_bytesRemaining == -1) || (remaining == 0) ){
for (int i = 0; i < _listeners.size(); i++)
- ((StatusListener)_listeners.get(i)).transferComplete(_alreadyTransferred, _bytesTransferred, _bytesRemaining, _url, _outputFile, _notModified);
+ ((StatusListener)_listeners.get(i)).transferComplete(
+ _alreadyTransferred,
+ _bytesTransferred,
+ _encodingChunked?-1:_bytesRemaining,
+ _url,
+ _outputFile,
+ _notModified);
} else {
throw new IOException("Disconnection on attempt " + _currentAttempt + " after " + _bytesTransferred);
}
@@ -387,6 +412,7 @@ public class EepGet {
switch (responseCode) {
case 200: // full
_out = new FileOutputStream(_outputFile, false);
+ _alreadyTransferred = 0;
rcOk = true;
break;
case 206: // partial
@@ -405,7 +431,7 @@ public class EepGet {
default:
rcOk = false;
}
-
+ buf.setLength(0);
byte lookahead[] = new byte[3];
while (true) {
int cur = _proxyIn.read();
@@ -435,7 +461,7 @@ public class EepGet {
if (!rcOk)
throw new IOException("Invalid HTTP response code: " + responseCode);
if (_encodingChunked) {
- readChunkLength();
+ _bytesRemaining = readChunkLength();
}
return;
}
@@ -450,7 +476,7 @@ public class EepGet {
}
}
- private void readChunkLength() throws IOException {
+ private long readChunkLength() throws IOException {
StringBuffer buf = new StringBuffer(8);
int nl = 0;
while (true) {
@@ -472,9 +498,9 @@ public class EepGet {
String len = buf.toString().trim();
try {
long bytes = Long.parseLong(len, 16);
- _bytesRemaining = bytes;
if (_log.shouldLog(Log.DEBUG))
_log.debug("Chunked length: " + bytes);
+ return bytes;
} catch (NumberFormatException nfe) {
throw new IOException("Invalid chunk length [" + len + "]");
}
@@ -529,6 +555,8 @@ public class EepGet {
} else if (key.equalsIgnoreCase("Transfer-encoding")) {
if (val.indexOf("chunked") != -1)
_encodingChunked = true;
+ } else if (key.equalsIgnoreCase("Content-Type")) {
+ _contentType=val;
} else {
// ignore the rest
}
@@ -630,5 +658,9 @@ public class EepGet {
public boolean getNotModified() {
return _notModified;
}
+
+ public String getContentType() {
+ return _contentType;
+ }
}
diff --git a/history.txt b/history.txt
index 4b1e7ddcca5637a8b79d9abef2cfdf618153209a..53ef9fd2f3bb4d8adfeb601f25e7c6f5c05aba31 100644
--- a/history.txt
+++ b/history.txt
@@ -1,4 +1,10 @@
-$Id: history.txt,v 1.308 2005/10/29 16:35:27 jrandom Exp $
+$Id: history.txt,v 1.309 2005/10/29 18:20:04 jrandom Exp $
+
+2005-10-30 dust
+ * Merge sucker into syndie with a rssimport.jsp page.
+ * Add getContentType() to EepGet.
+ * Make chunked transfer work (better) with EepGet.
+ * Do replaceAll("<","<") for logs.
* 2005-10-29 0.6.1.4 released
diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java
index 13e50dc206a7d5b1b0df09df6fd6f3c1a742ee3a..6cb5c124b2fd1239d0096e92523d7ea806b03846 100644
--- a/router/java/src/net/i2p/router/RouterVersion.java
+++ b/router/java/src/net/i2p/router/RouterVersion.java
@@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
*
*/
public class RouterVersion {
- public final static String ID = "$Revision: 1.279 $ $Date: 2005/10/29 16:35:24 $";
+ public final static String ID = "$Revision: 1.280 $ $Date: 2005/10/29 18:20:05 $";
public final static String VERSION = "0.6.1.4";
- public final static long BUILD = 0;
+ public final static long BUILD = 1;
public static void main(String args[]) {
System.out.println("I2P Router version: " + VERSION + "-" + BUILD);
System.out.println("Router ID: " + RouterVersion.ID);