1

我编写了代码来抓取网页并将图像保存在网页中。出于某种原因，我收到一个错误，我不确定如何修复。

我正在使用一种方法来确保我索引的每个图像都确实存在，所以我不确定为什么会发生这种情况。

这是我的代码：

import org.jsoup.Jsoup;
import org.jsoup.helper.Validate;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.net.*;
import java.awt.Image;
import java.awt.image.RenderedImage;
import java.io.*;

import java.io.IOException;

import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;

public class jsoup {
    public static void main(String[] args) throws IOException {
    crawl("http://www.istockphoto.com/photo");
}

public static void crawl(String crawlurl) throws IOException{
    Document doc = Jsoup.connect(crawlurl).get();
    getImgFromLinks(doc);
}

public static void getImgFromLinks(Document doc) throws IOException{
    Elements links = doc.select("a[href]");
    //System.out.println(links);

    for(int i=0;i<links.size();i++){
        if(exists(links.get(i).attr("href"))){
            System.out.println("crawled: " + links.get(i).attr("href"));
            getImages(doc, links.get(i).attr("href"));
        }else{
            System.out.println("I couldnt crawl: " + links.get(i).attr("href"));
        }
    }
}

public static String smartUrl(String url, String src) {
    if(exists(src)){
        return(src);
    }else{
        return(url + src);
    }
}


public static void getImages(Document doc, String url) throws IOException{



      for(int i=0; i<doc.getElementsByTag("img").size();i++){
            Element image = doc.select("img").get(i);
            String imgsrc = image.attr("src");
            if(imgsrc.toLowerCase().contains("png") || imgsrc.toLowerCase().contains("jpg") || imgsrc.toLowerCase().contains("jpeg") || imgsrc.toLowerCase().contains("gif")){

            int slashIndex = smartUrl(url, imgsrc).lastIndexOf('/');
            String finalUrl = smartUrl(url, imgsrc).substring(slashIndex);

            URL imgurl = new URL(smartUrl(url, imgsrc));

            if(exists(imgurl.toString())){
            Image crawledimg = ImageIO.read(imgurl);


            ImageIO.write((RenderedImage) crawledimg, "gif",new File("/Users/Jonathan/Desktop/crawledimages" + finalUrl));


            System.out.println("I got an image from:" + url + " Image Name: " + finalUrl);
            }

        }
        }


}


public static boolean exists(String URLName) {
    try {
      HttpURLConnection.setFollowRedirects(false);

    //HttpURLConnection.setInstanceFollowRedirects(false);
      HttpURLConnection con =
         (HttpURLConnection) new URL(URLName).openConnection();
      con.setRequestMethod("HEAD");
      return (con.getResponseCode() == HttpURLConnection.HTTP_OK);
    }
    catch (Exception e) {
       return false;
    }
  }
}

这是输出：

crawled: http://www.istockphoto.com/
I got an image from:http://www.istockphoto.com/ Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/ Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/ Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/ Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/ Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/ Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/ Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/ Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/ Image Name: /facebook.png
I got an image from:http://www.istockphoto.com/ Image Name: /twitter.png
I got an image from:http://www.istockphoto.com/ Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/ Image Name: /cartWhite.png
I couldnt crawl: #
I couldnt crawl: http://www.istockphoto.com/sign-in/aHR0cCUzQSUyRiUyRnd3dy5pc3RvY2twaG90by5jb20lMkZwaG90bw==
I couldnt crawl: http://www.istockphoto.com/join/aHR0cCUzQSUyRiUyRnd3dy5pc3RvY2twaG90by5jb20lMkZwaG90bw==
crawled: http://www.istockphoto.com/photo
I got an image from:http://www.istockphoto.com/photo Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/photo Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/photo Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/photo Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/photo Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/photo Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/photo Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/photo Image Name: /blank.gif
I got an image from:http://www.istockphoto.com/photo Image Name: /facebook.png
I got an image from:http://www.istockphoto.com/photo Image Name: /twitter.png
I got an image from:http://www.istockphoto.com/photo Image Name: /blank.gif
 Exception in thread "main" java.lang.IllegalArgumentException: im == null!
at javax.imageio.ImageIO.write(ImageIO.java:1457)
at javax.imageio.ImageIO.write(ImageIO.java:1527)
at jsoup.getImages(jsoup.java:68)
at jsoup.getImgFromLinks(jsoup.java:34)
at jsoup.crawl(jsoup.java:24)
at jsoup.main(jsoup.java:19)

图像将被保存，直到发生错误。

有人知道如何解决这个问题吗？

此外，由于某种原因，页面上的相同图像会被多次保存。

感谢您的时间，

乔纳森·奥伦。

4

1 回答 1

1

看起来像是null在里面通过ImageIO.write()

smartURL 函数有一个你需要解决的缺陷。它不是从网页获取的图像 URL 中创建预期的 URL。

例如： /static/images/cartWhite.png将由 smartURL 转换为http://www.istockphoto.com/photo/static/images/cartWhite.png不是图像但同时也不是错误页面。所以crawledimg指的是 null 导致IllegalArgumentException.

一个快速的解决方法是URL仅使用http://www.istockphoto.cominside创建getImages()。

图像将被保存多次，因为每个页面都有它们。您可以维护一个图像列表以避免这种情况发生。

我在您的代码中发现了另一个显示停止器，您将无法从您抓取的网页中检索任何其他图像。网站上的图片不以*.jpgor*.png等​​结尾。因此，您需要在开始之前研究网站上图片 URL 的模式。

于 2013-02-22T22:18:42.693 回答