被管理员和谐了的最高票答案“知乎数据抓取程序”（.net、c#数据挖掘）

问：能利用爬虫技术做到哪些很酷很有趣很有用的事情？

<span class="zm-entry-head-avatar-edit-button">修改头像</span>

base.WindowState = FormWindowState.Minimized;
base.Show();
base.Hide();
base.WindowState = FormWindowState.Normal;
base.ShowInTaskbar = false;
base.TopMost = false;
base.MaximizeBox = false;
base.MinimizeBox = false;
base.ControlBox = false;

//设置循环监测时间
int interval = int.Parse(Cinser.Common.ConfigurationHelper.GetAppSettingsValue("Interval"));
timer1.Interval = interval * 1000;

watchingURL = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("WatchingURL");
RunWhenStart(true, "zhApp.exe", """ + Application.StartupPath + "zhApp.exe" AutoRun");
Run();
dal.AddLog("程序初始化成功");
LoopCount += 1;
}

//设置程序开机自启动
public void RunWhenStart(bool Started, string name, string path)
{
RegistryKey HKLM = Registry.CurrentUser;
RegistryKey Run = HKLM.CreateSubKey(@"SOFTWAREMicrosoftWindowsCurrentVersionRun");
if (Started == true)
{
try
{
Run.SetValue(name, path);
HKLM.Close();
}
catch(Exception ex)//没有权限会异常
{
throw ex;
}
}
else
{
try
{
Run.DeleteValue(name);
HKLM.Close();
}
catch (Exception ex)//没有权限会异常
{
throw ex;
}
}
}

/// <summary>
/// 运行监测流程
/// </summary>
private void Run()
{
List<Question> questions = GetQuestions(Cinser.Common.HttpHelper.GetString(watchingURL));
string ids = dal.GetExistIdsStr();
for (int i = 0; i < questions.Count; i++)
{
if (ids.IndexOf(questions[i].Id) != ⑴)
{
questions.Remove(questions[i]);
i–;
}
else
{
if (ids == string.Empty)
ids = questions[i].Id;
else
ids += "," + questions[i].Id;
}
}
if (questions.Count > 0)
{
SendQuestions(questions);
dal.WriteIdStrToTxt(ids);
dal.Add(questions);
dal.AddLog(string.Format("获得了{0}条新数据。", questions.Count));
}
}

/// <summary>
/// 从监测站点源数据中抓取问题
/// </summary>
private List<Question> GetQuestions(string source)
{
List<Question> questions = new List<Question>();
string startStr = "<a class="question_link"";
if (source.IndexOf(startStr) != ⑴)
{
Question q = new Question();
string content = Cinser.Common.StringPlus.SubString(source, startStr, "</a>");
q.Id = Cinser.Common.StringPlus.SubString(content, "question/", """);
q.Title = content.Substring(content.IndexOf(">") + 1);
q.Time = DateTime.Now;
questions.Add(q);
source = source.Substring(source.IndexOf(startStr) + startStr.Length);
questions.AddRange(GetQuestions(source));
}
return questions;
}

private bool SendQuestions(List<Question> questions)
{
bool bSuccess = true;
List<string> strMailAddressTo = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("strMailAddressTo").Split(',').ToList();
string smtpAddress = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("smtpAddress");
string sendEmailFrom = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("sendEmailFrom");
string sendEmailFromPwd = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("sendEmailFromPwd");
string emailName = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("EmailName");
//密码解密，开源的话就去掉这步吧，这样配置的时候直接配置明文密码就行。
//sendEmailFromPwd = Cinser.Common.Security.DecryptDES(sendEmailFromPwd, "yuiophgf");
string msg = string.Empty;
SendCompletedEventHandler s = new SendCompletedEventHandler(SendCompleted);
string content = GetQustionsListStr(questions);
content += "

信息来源于：" + watchingURL;
Cinser.Common.SmtpEmailSend.SendEmail(strMailAddressTo, emailName + DateTime.Now.ToString(), content, smtpAddress, 0x19, sendEmailFrom, sendEmailFromPwd, "163测试邮箱", null, out msg, s);
return bSuccess;
}

private void SendCompleted(object sender, AsyncCompletedEventArgs e)
{
}

private string GetQustionsListStr(List<Question> questions)
{
string content = string.Empty;
if (questions != null && questions.Count > 0)
{
content = string.Format("名称：{0},url：{1}", questions[0].Title, questions[0].Url);
for (int i = 1; i < questions.Count; i++)
{
content += string.Format("
名称：{0},url：{1}", questions[i].Title, questions[i].Url);
}
}
return content;
}

private void timer1_Tick(object sender, EventArgs e)
{
int interval = int.Parse(Cinser.Common.ConfigurationHelper.GetAppSettingsValue("Interval"));
timer1.Interval = interval * 1000;
Run();
dal.AddLog(string.Format("程序循环次数：{0}", LoopCount++));
}
}

public string Remark
{
get { return remark; }
set { remark = value; }
}

public DateTime Time
{
get { return time; }
set { time = value; }
}

public string Type
{
get { return type; }
set { type = value; }
}

public string Url
{
get
{
if (string.IsNullOrEmpty(url))
{
url = string.Format("http://www.zhihu.com/question/{0}", Id);
}
return url;
}
set { url = value; }
}

public string Title
{
get { return title; }
set { title = value; }
}

public string Id
{
get { return id; }
set { id = value; }
}
}

public string DebugPath
{
get
{
if (debugPath == string.Empty)
debugPath = System.AppDomain.CurrentDomain.BaseDirectory;

if (debugPath.EndsWith("") == false)
{
debugPath += "";
}
return debugPath;
}
set { debugPath = value; }
}

public string LogPath
{
get
{
if (logPath == string.Empty)
{
logPath = DebugPath + "Log.txt";
}
return logPath;
}
}

public DataProvider()
{
dal = new Cinser.DBUtility.DAL.OracleDALCommon(this.ConnStr);
}

public string TxtPath
{
get
{
if (txtPath == string.Empty)
{
txtPath = DebugPath + "Ids.txt";
}
return txtPath;
}
}

public string ConnStr
{
get
{
if (connStr == string.Empty)
{
connStr = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("connStr");
}
return connStr;
}
set { connStr = value; }
}

public bool CanConnectOracleServer
{
get
{
return dal.Open();
}
}

/// <summary>
/// 将抓取到的问题写入oracle数据库中
/// </summary>
/// <param name="questions"></param>
/// <returns></returns>
public bool Add(List<Question> questions)
{
bool bReturn = false;
try
{
for (int i = 0; i < questions.Count; i++)
{
dal.Add("qustions", questions[i]);
}
bReturn = true;
}
catch { }
return bReturn;
}

public DataTable GetQustions(string sqlWhere = "1=1")
{
try
{
DataTable dt = dal.GetDataList("qustions", sqlWhere);
return dt;
}
catch
{
return null;
}
}

public bool IsExist(string id)
{
try
{
string sqlWhere = "id='" + id + "'";
DataTable dt = dal.GetDataList("qustions", sqlWhere);
return dt.Rows.Count > 0;
}
catch
{
return false;
}
}

/// <summary>
/// 获得已抓取过的问题ID字符串
/// </summary>
/// <returns></returns>
public string GetExistIdsStr()
{
string ids = string.Empty;
//如果能连上远程的oracle服务器则从oracle数据库中取ID字符串
if (CanConnectOracleServer)
{
DataTable dt = GetQustions();
if (dt != null && ᨲ鱸Ç૴Ç⚶ﭺ嫭蛿윺ꕎ腄衐⬤첁ꍪ쪀ྤ㐉妐揉䂲▱䧅䈩∧䤄閵ᢙꈴᦉ‴आᤩ꘤䤂ꒁ级鿩俐헕掱롸羖껡㺵��⎾ɂ꩒ሴ붤ﭽ粰髹｝￳忺磩ꄲ쿚㭅吺꺤絞⨏痦襸鞘詨韠⩳㕌蝹䴒壉턼骢⎪勰標째죶俶섾ﾟ䧸ﱛ瞴䯆ﱤ轧䭲ﳷ燱ﱺ쓻ᴗ﮿鴢╉ⴓ濴㑇튡玙����䏿㉳��躑짿��槰过￟矲勞旙ꓩ瓱녽��軿퇾俧ﲺ쯙閹습맵㙜宛熹䳄羡荷ꓔ턴籕��ﯳ꧆䣫빱ꛯ㴿㽆鿼ம藹��坢槨﹡慾Ꞵ��ㄕஏ靣︽Ὠ봯䟱爖駙쏼ﲮ⹟뷻ﻋ잵楱籘⓰횿羕춝ꏾ喇Ꮋ콁쩺㷋맧訯圗娩䴖낭嚼뮘宜��㑳煨굾歴鿞䢿穖䕨培拾⍋蓴댖줝㚶瞿缥맣滛ꇽ㐛展ⴛ῞Ꮄ렡⿤ꊾ渎쎛談콫��ﺇ㦝耗얹୥㐠ﯷ䩺䏝犑��ꈝ︉滼㬩迣팎଴緽觹뒧죓촂㼶騲ᵛ៷誖믐皁﫾繂糰��緊優ጏ㳽뾕䵵濤��寸ⓗ㍭ㇰ薀꓆朧ដﰖㇹꭧ럼᤯뵚︽꽴댿䃂绪샃忎䩾끫䝃哪毚髳䍉抝��矟⧕폠뿃穷麝뉽躻흎備쾣⯨鉺見淽㪝伛櫬ྰむ껅ꁮ꥜걫횸ᡛ尟䟴靮䝺敨䞅秧厉貣菣뒽뙐ﻹ홰绯ᾕ쟮⳴킐煿땾㊰䎻佇ḵￏ