----
presentation_topic: FEAR::API
presentation_title: FEAR::API - There's no fear with this elegant site scraper
presentation_place: 
presentation_date: 
----
== The assignment 

Please fetch the web pages from a certain commercial site, say Amazon.com.
Extract specifications of products from each page, and store each record in
database.

----
== What do we usually use?

* LWP::Simple
* LWP::UserAgent
* WWW::Mechanize
* Template::Extract
* Encode
* ....., etc.

----
== Example 1

           use LWP::Simple;
           use HTML::LinkExtractor;
           use Data::Dumper;

           my $input = get("http://google.com");
           my $LX = new HTML::LinkExtractor();

           $LX->parse(\$input);

           print Dumper($LX->links);

+

           # or you use WWW::Mechanize
           use WWW::Mechanize;
           use Data::Dumper;
           my $mech = WWW::Mechanize->new();
           $mech->get( "http://google.com" );
           print Dumper $mech->links;

----
== Example 2


           use WWW::Mechanize;
           use Data::Dumper;
           my $mech = WWW::Mechanize->new();
           $mech->get( "http://google.com" );
           my @link;
           foreach ($mech->links){
               if($_->[0] =~ /foo/){
                  $mech->get($_->[0]);
               }
               elsif($_->[0] =~ /bar/){
                  push @link;
               }
           }
           print Dumper \@link;

----
== Example 3

           use LWP::Simple;
           use Template::Extract;
           use Data::Dumper;
           my $obj = Template::Extract->new;
           print Dumper $obj->extract('[% FOREACH rec %]<a href="[% ... %]">[% link %]</a>[% END %]',
                                      get("http://google.com"));


----
== What do we need?

* An integrative module
* Less coding
* Something combining web page fetching and data extraction

+/So, FEAR::API was born!/

----
== Example 1 rewrite

=== Old
           use WWW::Mechanize;
           use Data::Dumper;
           my $mech = WWW::Mechanize->new();
           $mech->get( "http://google.com" );
           print Dumper $mech->links;


+=== New

 use FEAR::API -base;
 print Dumper fetch("google.com")->links;

----
== Example 2 rewrite

=== Old

           use WWW::Mechanize;
           use Data::Dumper;
           my $mech = WWW::Mechanize->new();
           $mech->get( "http://google.com" );
           my @link;
           foreach ($mech->links){
               if($_->[0] =~ /foo/){
                  $mech->get($_->[0]);
               }
               elsif($_->[0] =~ /bar/){
                  push @link;
               }
           }
           print Dumper \@link;

+=== New

 use FEAR::API -base;
 fetch("google.com") >> [
   qr(foo) => _self,
   qr(bar) => \my @link,
 ];
 $_->() while $_;
 print Dumper \@link;

----
== Example 3 rewrite

=== Old

           use LWP::Simple;
           use Template::Extract;
           use Data::Dumper;
           my $obj = Template::Extract->new;
           print Dumper $obj->extract('[% FOREACH rec %]<a href="[% ... %]">[% link %]</a>[% END %]',
                                      get("http://google.com"));
+=== New

 use FEAR::API -base;
 url("google.com")->();
 extract('<a href="[% ... %]">[% link %]</a>');
 print Dumper extresult;


----
== What is FEAR::API?

* FEAR::API is for:
+** Fetch, Extract, Aggregate, and Reorganize

* FEAR::API can help simplify the process of extracting data from web sites. It tries to help you
spend shorter time and write the least amount of code to come up to the same request.

+* Features:
** Combines the essence of LWP::Simple, WWW::Mechanize, Template::Extract, Encode
+** Cleaner code through operator overloading
+** Autoconverts web pages into UTF-8
+** Using FEAR::API is a bit like using a new sub-language, but with the full Perl support.
+** It's object-oriented.

+Let us see more examples

----
== Fetch a page and store it in a scalar

   use FEAR::API -base;

   my $content = fetch("google.com")->document->as_string;

   fetch("google.com") > my $content;

----
== Fetch a page and print to STDOUT

   getprint("google.com");

   print fetch("google.com")->document->as_string;

   fetch("google.com");
   print $$_;

   fetch("google.com") | _print;

----
== Fetch a page and save it to a file

   getstore("google.com");

   url("google.com")->() | _save_as("google.html");

   use IO::All;
   fetch("google.com") | io('google.html');


----
== Follow links in Google's homepage

   url("google.com")->() >> _self;
   &$_ while $_;

----
== Save links in Google's homepage

    url("google.com")->() >> _self | _save_as_tree("./root");
    $_->() | _save_as_tree("./root") while $_;


----
== Recursively get web pages from Google

    url("google.com");
    &$_ >> _self while $_;

----
== Recursively get web pages from Google

    url("google.com");
    &$_ >> _self | _save_as_tree("./root") while $_;

----
== Follow the second link of Google

    url("google.com")->()->follow_link(n => 2);

----
== Return links from Google's homepage

    print Dumper fetch("google.com")->links;

----
== Submit a query to Google

    url("google.com")->();
    submit_form(
                form_number => 1,
                fields => { q => "Kill Bush" }
                );

----
== Get links of some pattern

    url("[% FOREACH i = ['a'..'z'] %]
         http://some.site/[% i %]
         [% END %]");
    &$_ while $_;

----
== Deal with links in a web page (I)

    url("google.com")->()
      >> [
          qr(^http:) => _self,
          qr(google) => \my @l,
          qr(google) => sub {  print ">>>".$_[0]->[0],$/ }
         ];
    $_->() while $_;
    print Dumper \@l;

----
== Deal with links in a web page (II)

    url("google.com")->()
      >> {
          qr(^http:) => _self,
          qr(google) => \my @l,
          qr(google) => sub {  print ">>>".$_[0]->[0],$/ }
         };
    $_->() while $_;
    print Dumper \@l;

----
== Get pages in parallel

    url("google.com")->() >> _self;
    pfetch(sub{
               local $_ = shift;
               print join q/ /, title, current_url, document->size, $/;
           });


----
== Extraction

=== Extract data from CPAN

    url("http://search.cpan.org/recent")->();
    submit_form(
            form_name => "f",
            fields => {
                       query => "perl"
                      });
    template("<!--item-->[% p %]<!--end item-->");
    extract;
    print Dumper extresult;

+=== Extract data from CPAN after some HTML cleanup

    url("http://search.cpan.org/recent")->();
    submit_form(
            form_name => "f",
            fields => {
                       query => "perl"
                      });
    preproc(q(s/\A.+<!--results-->(.+)<!--end results-->.+\Z/$1/s));
    print document->as_string;    # print content to STDOUT
    template("<!--item-->[% p %]<!--end item-->");
    extract;
    print Dumper extresult;

+=== HTML cleanup, extract data, and refine results

    url("http://search.cpan.org/recent")->();
    submit_form(
            form_name => "f",
            fields => {
                       query => "perl"
                      });
    preproc(q(s/\A.+<!--results-->(.+)<!--end results-->.+\Z/$1/s));
    print $$_;    # print content to STDOUT
    template("<!--item-->[% rec %]<!--end item-->");
    extract;
    postproc(q($_->{rec} =~ s/<.+?>//g));     # Strip HTML tags
    print Dumper extresult;


+=== Use filtering syntax

    fetch("http://search.cpan.org/recent");
    submit_form(
                form_name => "f",
                fields => {
                           query => "perl"
                });
    $_ | _doc_filter(q(s/\A.+<!--results-->(.+)<!--end results-->.+\Z/$1/s))
       | _template("<!--item-->[% rec %]<!--end item-->")
       | _result_filter(q($_->{rec} =~ s/<.+?>//g));
    print Dumper \@$_;

+=== Invoke handler for extracted results

    fetch("http://search.cpan.org/recent");
    submit_form(
                form_name => "f",
                fields => {
                           query => "perl"
                });
    $_ | _doc_filter(q(s/\A.+<!--results-->(.+)<!--end results-->.+\Z/$1/s))
       | "<!--item-->[% rec %]<!--end item-->"
       | _result_filter(q($_->{rec} =~ s/<.+?>//g));
    invoke_handler('Data::Dumper');

+
You can also put extracted results straight into relational databases.

    invoke_handler('Some::Module::based::on::Class::DBI');

    invoke_handler('Some::Module::based::on::DBIx::Class::CDBICompat');

----
== Preprocess document

    url("google.com")->()
    | _preproc(use => "html_to_null")
    | _preproc(use => "decode_entities")
    | _print;

----
== Postprocess extraction results

    fetch("http://search.cpan.org/recent");
    submit_form(
                form_name => "f",
                fields => {
                           query => "perl"
                });
    $_ | _doc_filter(q(s/\A.+<!--results-->(.+)<!--end results-->.+\Z/$1/s))
       | _template("<!--item-->[% rec %]<!--end item-->")
       | _result_filter(use => "html_to_null",    qw(rec));
       | _result_filter(use => "decode_entities", qw(rec))
    print Dumper \@$_;

----
== Shell-like functions

   file("some.document.txt")->()
   | _map(qr(some_pattern))
   | _sort
   | _grep(qr(some_other_pattern))
   | _uniq
   ;

----
== If you don't use with -base

   use FEAR::API;
   my $f = fear();
   $f->fetch("blah");
   # ..., etc.


Use with -base, and $_ will be set to FEAR::API.

   use FEAR::API -base;
   fetch("blah");
   # ..., etc.

----
== Use FEAR::API in command line

    fearperl -e 'fetch("google.com")'

    perl -M'FEAR::API -base' -e 'fetch("google.com")'

----
== TO DO

* Code refactoring and cleaning
* Add more tests
* Patch documents
* Develop more methods

----
banner_bgcolor: lightblue
----
== That's All:D

* The END