java网页数据抓取实例

网页上面数据如下：

如果想要过去上图所示网页的数据，代码如下：

（1）调度类，主要调用工具类中的方法获取数据并入库

package com.jointsky.jointframe.weather.jobservice;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.springframework.transaction.annotation.Transactional;

import com.jointsky.jointframe.scheduler.exception.ExecutionException;
import com.jointsky.jointframe.scheduler.quartz.JobService;
import com.jointsky.jointframe.weather.entity.ActuallyForecastWeather;
import com.jointsky.jointframe.weather.entity.ActuallyWeather;
import com.jointsky.jointframe.weather.service.ActuallyForecastWeatherManager;
import com.jointsky.jointframe.weather.service.ActuallyWeatherManager;
import com.jointsky.jointframe.weather.utils.UrlInfo;

/*
 * <p>Description:实况天气资料数据资料调度类</p>*/
@Transactional
public class ActuallyWeatherJobService implements JobService{
    
    /**
     * 实况天气资料（当天）数据管理类
     */
    private ActuallyWeatherManager actuallyWeatherManager;
    
    /**
     * 实况天气资料（当天）数据实体类
     */
    private ActuallyWeather actuallyWeather;
    
    /**
     * 实况天气资料预报数据管理类
     */
    private ActuallyForecastWeatherManager actuallyForecastWeatherManager;
    
    /**
     * 实况天气资料预报数据实体类
     */
    private ActuallyForecastWeather actuallyForecastWeather;
    
    /**
     * 存放参数的map集合
     */
    private Map<String, Object> map = new HashMap<String, Object>();
    
    

    @Override
    public void execute(Map<String, Object> arg0) throws ExecutionException {
        System.out.println("实况天气资料数据获取调度成功");
        String[] countyCodes = UrlInfo.getCountyCodes().split(",");
        for (int j = 0; j < countyCodes.length; j++) {
            String countyCode = countyCodes[j];
            if (StringUtils.isNotEmpty(countyCode)) {
                try {
                    String url = UrlInfo.getUrl(countyCode);
                    Map<String, Object> filterMap = UrlInfo.getDistrict(countyCode);
                    //市级名称
                    String cityLevel = (String) filterMap.get("cityLevel");
                    //区县级名称
                    String countyLevel = (String) filterMap.get("countyLevel");
                    //银川天气预报详细信息数据
                    List<ActuallyWeather> list_actually = UrlInfo.getURLInfoOfActully(url,"utf-8");
                    List<ActuallyForecastWeather> list_forecast = UrlInfo.getURLInfoOfForecast(url, "utf-8");
                    for (int i = 0; i < list_actually.size(); i++) {
                        actuallyWeather = list_actually.get(i);
                        actuallyWeather.setCityLevel(cityLevel);
                        actuallyWeather.setCountyLevel(countyLevel);
                        //预报时间
                        if (StringUtils.isNotEmpty(actuallyWeather.getForecastTime())) {
                            map.put("forecastTime", actuallyWeather.getForecastTime());
                        }
                        //地名
                        if (StringUtils.isNotEmpty(actuallyWeather.getPlaceName())) {
                            map.put("placeName", actuallyWeather.getPlaceName());
                        }
                        String actuallyWeatherId = actuallyWeatherManager.findIdByParams(map);
                        map = new HashMap<String, Object>();
                        if (StringUtils.isNotEmpty(actuallyWeatherId)) {
                            actuallyWeather.setId(actuallyWeatherId);
                            actuallyWeatherManager.updateWeather(actuallyWeather);
                        }else {
                            actuallyWeatherManager.save(actuallyWeather);
                        }
                    }
                    for (int i = 0; i < list_forecast.size(); i++) {
                        actuallyForecastWeather = list_forecast.get(i);
                        actuallyForecastWeather.setCityLevel(cityLevel);
                        actuallyForecastWeather.setCountyLevel(countyLevel);
                        //预报时间
                        if (StringUtils.isNotEmpty(actuallyForecastWeather.getForecastTime())) {
                            map.put("forecastTime", actuallyForecastWeather.getForecastTime());
                        }
                        //地名
                        if (StringUtils.isNotEmpty(actuallyForecastWeather.getPlaceName())) {
                            map.put("placeName", actuallyForecastWeather.getPlaceName());
                        }
                        String actuallyForecastWeatherId = actuallyForecastWeatherManager.findIdByParams(map);
                        map = new HashMap<String, Object>();
                        if (StringUtils.isNotEmpty(actuallyForecastWeatherId)) {
                            actuallyForecastWeather.setId(actuallyForecastWeatherId);
                            actuallyForecastWeatherManager.updateForecastWeather(actuallyForecastWeather);
                        }else {
                            actuallyForecastWeatherManager.save(actuallyForecastWeather);
                        }
                    }
                    
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }

    

    public ActuallyWeatherManager getActuallyWeatherManager() {
        return actuallyWeatherManager;
    }

    public void setActuallyWeatherManager(
            ActuallyWeatherManager actuallyWeatherManager) {
        this.actuallyWeatherManager = actuallyWeatherManager;
    }

    public ActuallyWeather getActuallyWeather() {
        return actuallyWeather;
    }

    public void setActuallyWeather(ActuallyWeather actuallyWeather) {
        this.actuallyWeather = actuallyWeather;
    }



    public Map<String, Object> getMap() {
        return map;
    }



    public void setMap(Map<String, Object> map) {
        this.map = map;
    }



    public ActuallyForecastWeatherManager getActuallyForecastWeatherManager() {
        return actuallyForecastWeatherManager;
    }



    public void setActuallyForecastWeatherManager(
            ActuallyForecastWeatherManager actuallyForecastWeatherManager) {
        this.actuallyForecastWeatherManager = actuallyForecastWeatherManager;
    }



    public ActuallyForecastWeather getActuallyForecastWeather() {
        return actuallyForecastWeather;
    }



    public void setActuallyForecastWeather(
            ActuallyForecastWeather actuallyForecastWeather) {
        this.actuallyForecastWeather = actuallyForecastWeather;
    }


    
    
}

View Code