Building a Web Scraper without Dream

Uri nyt_uri = new Uri("http://query.nytimes.com/");
UriBuilder uri_build = new UriBuilder(nyt_uri);

Now we add the path:

uri_build.Path = "search/query";

...encode the user_input and then add the query string:

string encode_input = HttpUtility.UrlEncode(user_input);
uri_build.Query = "query=" + encode_input + "&srchst=nyt&n=100;

XUri nyt_xuri = new XUri("http://query.nytimes.com/");
XUri nyt_full_uri = nyt_xuri.At("search", "query").With("query", user_input).With("srchst", "nyt").With("n", "100");

First, generate a temporary file in which we will store our result in (you will need this later):

string fname = Path.GetTempFileName();
string filename = fname + ".xml";

Next, we create the WebRequest and then specify that you're using HTTP GET. After we've done this, send the request with the GetResponse() call and handle the response as a WebResponse:

WebRequest request = WebRequest.Create(uri_build.Uri);
request.Method = "GET";
using(WebResponse response = request.GetResponse()) {

Since we can't use the WebResponse in it's raw form, we need to hand it to a StreamReader class:

using(StreamReader reader = new StreamReader(response.GetResponseStream())) {

Moreover, because we are using the Sgml class that we added to our reference earlier, we now need to initalize the SgmlReader and pass it the StreamReader we just created:

Sgml.SgmlReader sgmlReader = new Sgml.SgmlReader();
sgmlReader.DocType = "html";
sgmlReader.WhitespaceHandling = WhitespaceHandling.All;
sgmlReader.CaseFolding = Sgml.CaseFolding.ToLower;
sgmlReader.InputStream = reader;

Plug plug = Plug.New(nyt_full_uri);
DreamMessage message = plug.Get();

We initalize the XmlTextWriter by passing it the temporary file we created earlier into XmlTextWriter's constructor:

Encoding unicode = Encoding.UTF8; //set XML encoding
using(XmlTextWriter xtw = new XmlTextWriter(filename, unicode)) {

xtw.WriteStartDocument(false);

xtw.WriteStartElement("html");
xtw.WriteStartElement("head");
xtw.WriteElementString("title", user_input);
xtw.WriteEndElement();
xtw.WriteStartElement("body");

xtw.WriteElementString("h3", "Search Results for: " + user_input);
xtw.WriteStartElement("table");
xtw.WriteAttributeString("border", "1");

XDoc output = new XDoc("html"); //html wrapper
output.Start("head").Elem("title", user_input).End();
output.Start("body");
output.Elem("h3", "Search Result for: " + user_input);
output.Start("table").Attr("border", "1");// table for results

Note: we don't need to create a temporary file until later, after we're done and ready to flush the entire Xml document to a file.

Create an XmlDocument, to help you handle the xpath needed to traverse to the content you wish to extract from the response HTML:

try {
    XmlDocument doc = new XmlDocument();
    doc.PreserveWhitespace = true;
    doc.XmlResolver = null;
    doc.Load(sgmlReader);

    XmlNode xroot = doc.DocumentElement;

    // traverse the path to the particular list element you want to look at
    XmlNodeList nList = xroot.SelectNodes("body/div/div/div/ol/li");

Parse the main content for article title/link, aritcle snipplet, and article information:

foreach(XmlNode xNode in nList) {
    xtw.WriteStartElement("tr");
    xtw.WriteStartElement("td");
    XmlNode h3 = xNode["h3"]; // find title/link to article
    if(h3 != null) {
        xtw.WriteNode(new XmlNodeReader(h3), true);
    }
    XmlNode p = xNode["p"];// find article snipplet
    if(p != null) {
        xtw.WriteNode(new XmlNodeReader(p), true);
    }
    XmlNode div = xNode["div"];// find article information
    if(div != null) {
        xtw.WriteNode(new XmlNodeReader(div), true);
    }
    xtw.WriteEndElement(); // close td
    xtw.WriteEndElement(); // close tr
}
} catch(Exception e) {
    Console.WriteLine(e);
    return;
}

Finally, we close the tags, flush the content out into our prepared file and then close the file:

xtw.WriteEndElement(); // close table
    xtw.WriteEndElement(); // close body
    xtw.WriteEndElement(); // close html
    xtw.Flush();
    xtw.Close();
}

XDoc doc = message.AsDocument();
foreach(XDoc entry in doc["body/div/div/div/ol/li"]) {
    output.Start("tr").Start("td");
    output.Add(entry["h3"]);    // retrieve article title and link
    output.Add(entry["p"]);     // retrieve article snipplet
    output.Add(entry["div"]);   // retrieve article info(author, date, number of words)
    output.End().End();         // close off tr and td
}

Close the end of the table and the body

output.End().End();
}

Create a temporary file and flush the result out to it in XHTML format

string filename = Path.GetTempFileName()+".html";
    File.WriteAllText(filename, output.ToXHtml());
}

Create a method to LaunchProgram and call it in HandleSearch:

LaunchProgram(fname + ".html");

The contents of our LaunchProgram method consists of:

public void LaunchProgram(string fname) {

    // create a new process
    System.Diagnostics.Process proc = new System.Diagnostics.Process();
    proc.StartInfo.FileName = "explorer";
    proc.StartInfo.Arguments = fname;
    proc.Start();
}

We request the operating system to open a browser and execute the file with the screen scraped article list we just created

Async.ExecuteProcess("explorer.exe", filename, Stream.Null, new Result<Tuple<int, Stream, Stream>>());