Index: Main.java
===================================================================
RCS file: /home/cvspublic/xml-cocoon2/src/org/apache/cocoon/Main.java,v
retrieving revision 1.4
diff -u -r1.4 Main.java
--- Main.java	2001/05/31 17:37:31	1.4
+++ Main.java	2001/06/17 03:27:52
@@ -16,6 +16,7 @@
 import java.io.OutputStream;
 import java.io.PrintStream;
 import java.net.MalformedURLException;
+import java.util.HashSet;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -118,7 +119,7 @@
 
             switch (option.getId()) {
                 case 0:
-                    targets.add(option.getArgument());
+                    targets.add(NetUtils.normalize(option.getArgument()));
                     break;
 
                 case Main.HELP_OPT:
@@ -299,6 +300,9 @@
     private File destDir;
     private File context;
     private Map attributes;
+    private HashMap empty;
+    private Map allProcessedLinks;
+    private Map allTranslatedLinks;
 
     /**
      * Creates the Main class
@@ -308,6 +312,9 @@
         this.context = context;
         this.destDir = destDir;
         this.attributes = new HashMap();
+        this.empty = new HashMap();
+        this.allProcessedLinks = new HashMap();
+        this.allTranslatedLinks = new HashMap();
     }
 
     /**
@@ -326,13 +333,24 @@
     public int process(Collection uris, boolean xspOnly) throws Exception {
         int nCount = 0;
         log.info("...ready, let's go:");
-        Iterator i = uris.iterator();
-        while (i.hasNext()) {
-            if(xspOnly)
-                this.processXSP(NetUtils.normalize((String) i.next()));
-            else
-                this.processURI(NetUtils.normalize((String) i.next()), 0);
+
+        HashSet links = new HashSet();
+        links.addAll(uris);
+        while (links.size() > 0) {
+        	Iterator i = links.iterator();
+           	String url = (String)i.next();
+           	if(allProcessedLinks.get(url) == null){
+                if(xspOnly){
+                    this.processXSP(url);
+                }else{
+                    links.addAll(this.processURI(url));
+                }
+	        }
+            links.remove(url);
             nCount++;
+
+System.err.println("  Memory: " + (Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory()));
+System.err.println("  Processed, Translated & Left: " + allProcessedLinks.size() + ", "  + allTranslatedLinks.size() + ", " + links.size());
         }
         return nCount;
     }
@@ -361,61 +379,107 @@
     }
 
     /**
-     * Processes the given URI in a recursive way. The algorithm followed by
-     * this method is the following:
+     * Processes the given URI and return all links. The algorithm is the following:
      *
      * <ul>
-     *  <li>the link view of the given URI is called and the resources linked
-     *      to the requested one are obtained.</li>
-     *  <li>for each link, this method is recursively called and returns
-     *      the file used to save the resource on disk.</li>
+     *  <li>file name for the URI is generated. URI MIME type is checked for
+     *      consistency with the URI and, if the extension is inconsistent
+     *      or absent, the file name is changed</li>
+     *  <li>the link view of the given URI is called and the file names for linked
+     *      resources are generated and stored.</li>
+     *  <li>for each link, absolute file name is translated to relative path.</li>
      *  <li>after the complete list of links is translated, the link-translating
      *      view of the resource is called to obtain a link-translated version
      *      of the resource with the given link map</li>
-     *  <li>the resource is saved on disk and the URI MIME type is checked for
-     *      consistency with the URI and, if the extension is inconsistent
-     *      or absent, the file is renamed</li>
-     *  <li>then the file name of the translated URI is returned</li>
+     *  <li>list of absolute URI is returned, for every URI which is not yet
+     *      present in list of all translated URIs</li>
      * </ul>
      */
-    public String processURI(String uri, int level) throws Exception {
-        log.info("Processing URI: " + leaf(level) + uri);
+    public Collection processURI(String uri) throws Exception {
+        log.info("Processing URI: " + uri);
+
+	    // Get parameters, deperameterized URI and path from URI
+        final HashMap parameters = new HashMap();
+        parameters.put("user-agent", Constants.COMPLETE_NAME);
+        parameters.put("accept", "*");
+        final String deparameterizedURI = NetUtils.deparameterize(uri, parameters);
+        final String path = NetUtils.getPath(uri);
+
+        // Get file name from URI (without path)
+        String pageURI = deparameterizedURI;
+        if(pageURI.indexOf("/") != -1){
+	        pageURI = pageURI.substring(pageURI.lastIndexOf("/") + 1);
+	        if(pageURI.length() == 0) pageURI = "./";
+	    }
+
+        String filename = (String)allTranslatedLinks.get(uri);
+        if(filename == null){
+            filename = mangle(uri);
+            final String type = getType(deparameterizedURI, parameters);
+            final String ext = NetUtils.getExtension(filename);
+            final String defaultExt = MIMEUtils.getDefaultExtension(type);
+            if ((ext == null) || (!ext.equals(defaultExt))) {
+                filename += defaultExt;
+            }
+            allTranslatedLinks.put(uri, filename);
+        }
+
+        // Store processed URI list to avoid eternal loop
+        allProcessedLinks.put(uri, filename);
 
-        Collection links = this.getLinks(uri);
-        Map translatedLinks = new HashMap(links.size());
-        Iterator i = links.iterator();
+        // Process links
+        final ArrayList absoluteLinks = new ArrayList();
+        final HashMap translatedLinks = new HashMap();
+        final Iterator i = this.getLinks(deparameterizedURI, parameters).iterator();
         while (i.hasNext()) {
-            log.info(tree(level));
-            String path = NetUtils.getPath(uri);
             String relativeLink = (String) i.next();
-            String absoluteLink = NetUtils.normalize(NetUtils.absolutize(path, relativeLink));
-            String translatedAbsoluteLink = this.processURI(absoluteLink, level + 1);
-            String translatedRelativeLink = NetUtils.relativize(path, translatedAbsoluteLink);
+            // Fix relative links starting with "?"
+            if(relativeLink.startsWith("?")){
+            	relativeLink = pageURI + relativeLink;
+            }
+
+            final String absoluteLink = NetUtils.normalize(NetUtils.absolutize(path, relativeLink));
+            String translatedAbsoluteLink = (String)allTranslatedLinks.get(absoluteLink);
+            if(translatedAbsoluteLink == null){
+                translatedAbsoluteLink = this.translateURI(absoluteLink);
+                log.info("  Link translated: " + absoluteLink);
+                allTranslatedLinks.put(absoluteLink, translatedAbsoluteLink);
+                absoluteLinks.add(absoluteLink);
+            }
+
+            final String translatedRelativeLink = NetUtils.relativize(path, translatedAbsoluteLink);
             translatedLinks.put(relativeLink, translatedRelativeLink);
         }
 
-        String filename = mangle(uri);
+        // Process URI
         File file = IOUtils.createFile(destDir, filename);
         OutputStream output = new BufferedOutputStream(new FileOutputStream(file));
-        String type = getPage(uri, translatedLinks, output);
+        String type = getPage(deparameterizedURI, parameters, translatedLinks, output);
         output.close();
 
+        if (type == null) {
+            log.warn("  [broken link]--> " + filename);
+            resourceUnavailable(file);
+        } else {
+            log.info("  [" + type + "]--> " + filename);
+        }
+
+        return absoluteLinks;
+    }
+
+    public String translateURI(String uri) throws Exception {
+        HashMap parameters = new HashMap();
+        parameters.put("user-agent", Constants.COMPLETE_NAME);
+        parameters.put("accept", "*");
+        String deparameterizedURI = NetUtils.deparameterize(uri, parameters);
+
+        String path = NetUtils.getPath(uri);
+        String filename = mangle(uri);
+        String type = getType(deparameterizedURI, parameters);
         String ext = NetUtils.getExtension(filename);
         String defaultExt = MIMEUtils.getDefaultExtension(type);
-
         if ((ext == null) || (!ext.equals(defaultExt))) {
             filename += defaultExt;
-            File newFile = IOUtils.createFile(destDir, filename);
-            file.renameTo(newFile);
-            file = newFile;
-        }
-        log.info(tree(level));
-
-        if (type == null) {
-            log.warn(leaf(level + 1) + "[broken link]--> " + filename);
-            resourceUnavailable(file);
-        } else {
-            log.info(leaf(level + 1) + "[" + type + "]--> " + filename);
         }
 
         return filename;
@@ -455,17 +519,27 @@
         return buffer.toString();
     }
 
-    Collection getLinks(String uri) throws Exception {
-        HashMap parameters = new HashMap();
-        String deparameterizedURI = NetUtils.deparameterize(uri, parameters);
+    Collection getLinks(String deparameterizedURI, Map parameters) throws Exception {
         LinkSamplingEnvironment env = new LinkSamplingEnvironment(deparameterizedURI, context, attributes, parameters);
         cocoon.process(env);
         return env.getLinks();
     }
 
-    String getPage(String uri, Map links, OutputStream stream) throws Exception {
-        HashMap parameters = new HashMap();
-        FileSavingEnvironment env = new FileSavingEnvironment(uri, context, attributes, parameters, links, stream);
+    String getPage(String deparameterizedURI, Map parameters, Map links, OutputStream stream) throws Exception {
+        FileSavingEnvironment env = new FileSavingEnvironment(deparameterizedURI, context, attributes, parameters, links, stream);
+        cocoon.process(env);
+        return env.getContentType();
+    }
+
+    static class NullOutputStream extends OutputStream
+    {
+	    public void write(int b) throws IOException { }
+	    public void write(byte b[]) throws IOException { }
+	    public void write(byte b[], int off, int len) throws IOException { }
+    }
+
+    String getType(String deparameterizedURI, Map parameters) throws Exception {
+        FileSavingEnvironment env = new FileSavingEnvironment(deparameterizedURI, context, attributes, parameters, empty, new NullOutputStream());
         cocoon.process(env);
         return env.getContentType();
     }