overture scrapper - perl

spiderninja · 03-30-2007, 07:08 PM

PHP Code:

  use strict;

use IO::Socket;

$| = 1;

 
# http://www.spiderninja.com/

# ninja@spiderninja.com

 
# COUNTRY CODES

# -------------

# Australia      => 'au'

# Austria        => 'at'

# Denmark        => 'dk'

# Finland        => 'fi'

# France         => 'fr'

# Germany        => 'de'

# Italia         => 'it'

# Netherlands    => 'nl'

# Norway         => 'no'

# Spain          => 'es'

# Sweden         => 'se'

# Switzerland    => 'ch'

# United Kingdom => 'uk'

# United States  => 'us'

 
my $mkt = "de";

my @keywords = qw(spider ninja);

my $include_count = 0;

 
for my $keyword (@keywords) {

    get_overture_data($mkt, $keyword, $include_count);

    sleep 1;

}

 
sub get_overture_data {

    my ($mkt, $keyword, $include_count) = @_;

    my $socket = IO::Socket::INET->new(PeerAddr => "inventory.overture.com",

                                       PeerPort => 80,

                                       Proto    => "tcp",

                                       Type     => SOCK_STREAM,

                                       Timeout  => 5);

    if (!$socket) {

????: NamePros.com http://www.namepros.com/code/311235-overture-scrapper-perl.html
        print "couldn't connect to overture";

        exit;

    }

    my $request = get_request($keyword, $mkt);

    print $socket $request;

 
    while (<$socket>) {

        # grab keyword

        if (/color=#000000>(.*)<\/a><\/td>/) {

            print "$1\n";

        }

        # grab number

        if ($include_count) {

            if (/size=1>\&nbsp;(.*)<\/td>/) {

                print "$1 - ";

            }

        }

        # special case where suggestion equals search term

        if (/color=E8E8E8>&nbsp\;(.*)<\/a><\/td>/) {

            print "$1\n";

        }

        # grab number for special case

        if ($include_count) {

            if (/color=E8E8E8>\&nbsp;(\d+)/) {

                print "$1 - ";

            }

        }

        # or...note if nothing is there

        if (/<em>(No suggestions for .*)<\/em>/) {

            print "$1\n";

        }

        last if /<\/html>/;

    }

    close $socket;

    return;

}

 
sub get_request {

    my ($keyword, $mkt) = @_;

????: NamePros.com http://www.namepros.com/showthread.php?t=311235
    my $post_request = "mkt=$mkt&term=$keyword";

    my $length = length($post_request);

 
    my $http_request = qq{POST /d/searchinventory/suggestion/ HTTP/1.0

Host: inventory.overture.com

Accept: text/html, text/plain

Accept-Encoding: gzip

Accept-Language: en

User-Agent: Lynx/2.8.3rel.1 libwww-FM/2.14

Referer: http://inventory.overture.com/d/searchinventory/suggestion/

Content-type: application/x-www-form-urlencoded

Content-length: $length

 
$post_request};

 
    return $http_request;

}

nice tool that does what you want it to do ;-)

03-30-2007, 07:08 PM	THREAD STARTER #1 (permalink)
spiderninja NamePros Member Join Date: Dec 2005 Posts: 43	overture scrapper - perl PHP Code: use strict; use IO::Socket; $\| = 1; # http://www.spiderninja.com/ # ninja@spiderninja.com # COUNTRY CODES # ------------- # Australia => 'au' # Austria => 'at' # Denmark => 'dk' # Finland => 'fi' # France => 'fr' # Germany => 'de' # Italia => 'it' # Netherlands => 'nl' # Norway => 'no' # Spain => 'es' # Sweden => 'se' # Switzerland => 'ch' # United Kingdom => 'uk' # United States => 'us' my $mkt = "de"; my @keywords = qw(spider ninja); my $include_count = 0; for my $keyword (@keywords) { get_overture_data($mkt, $keyword, $include_count); sleep 1; } sub get_overture_data { my ($mkt, $keyword, $include_count) = @_; my $socket = IO::Socket::INET->new(PeerAddr => "inventory.overture.com", PeerPort => 80, Proto => "tcp", Type => SOCK_STREAM, Timeout => 5); if (!$socket) { ????: NamePros.com http://www.namepros.com/code/311235-overture-scrapper-perl.html print "couldn't connect to overture"; exit; } my $request = get_request($keyword, $mkt); print $socket $request; while (<$socket>) { # grab keyword if (/color=#000000>(.)<\/a><\/td>/) { print "$1\n"; } # grab number if ($include_count) { if (/size=1>\ (.)<\/td>/) { print "$1 - "; } } # special case where suggestion equals search term if (/color=E8E8E8>&nbsp\;(.)<\/a><\/td>/) { print "$1\n"; } # grab number for special case if ($include_count) { if (/color=E8E8E8>\ (\d+)/) { print "$1 - "; } } # or...note if nothing is there if (/<em>(No suggestions for .)<\/em>/) { print "$1\n"; } last if /<\/html>/; } close $socket; return; } sub get_request { my ($keyword, $mkt) = @_; ????: NamePros.com http://www.namepros.com/showthread.php?t=311235 my $post_request = "mkt=$mkt&term=$keyword"; my $length = length($post_request); my $http_request = qq{POST /d/searchinventory/suggestion/ HTTP/1.0 Host: inventory.overture.com Accept: text/html, text/plain Accept-Encoding: gzip Accept-Language: en User-Agent: Lynx/2.8.3rel.1 libwww-FM/2.14 Referer: http://inventory.overture.com/d/searchinventory/suggestion/ Content-type: application/x-www-form-urlencoded Content-length: $length $post_request}; return $http_request; } nice tool that does what you want it to do ;-) __________________ Spider Ninja's free SEO tools

Currently Active Users Viewing This Thread: 1 (0 members and 1 guests)