模拟爬虫可访问

package Ntework;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
/**
 * 1.URL
 * 2.下载
 * -----
 * 3.分析
 * 4.正则表达式
 * 5.数据存储
 * 6.数据清洗
 * @author pmc
 *
 */
public class URLTest2 {
	public static void main(String[] args) throws Exception{
		//获取URL
		URL url=new URL("https://www.dianping.com");
		//下载资源
		InputStream is=url.openStream();
		BufferedReader bf=new BufferedReader(new InputStreamReader(is,"UTF-8"));
		String msg=null;
		while((msg=bf.readLine())!=null){
			System.out.println(msg);
		}
		bf.close();
		
		
	}
}

不可访问,采用模拟浏览器,HttpURLConnection,设置参数

conn.setRequestMethod("GET");
        conn.setRequestProperty("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1");

package Ntework;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

/**
 * 网络爬虫的原理+模拟浏览器
 * get从服务器获取资源(不安全)
 * post提交数据(安全)
 * @author pmc
 *
 */
public class URLTest3 {
	public static void main(String[] args) throws Exception {
		// 获取URL
		URL url = new URL("https://www.dianping.com");
		// 下载资源+模拟浏览器
		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
		conn.setRequestMethod("GET");
		conn.setRequestProperty("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1");
		// InputStream is=url.openStream();
		BufferedReader bf = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"));
		String msg = null;
		while ((msg = bf.readLine()) != null) {
			System.out.println(msg);
		}
		bf.close();

	}
}