HTTrack Website Copier
Free software offline browser - FORUM
Subject: OFF Topic: HTTP protocol problem.
Author: Denis Labon
Date: 12/20/2002 21:25
 
This is a off topic message and has nothing to do with
httrack. I'm desperate and need help. Maybe someone can help me.

My situation:
I try to make a simple program to retrieve data from a
webpage and store it in a file called "t.htm" but it only
works for certain websites and I don't know why.
I test it with www.yahoo.com and it works but not for
www.altavista.com.

The program is written in C/C++ under Linux with g++ compiler.
Program execute like the following: ./testing www.yahoo.com
Here is the code:

/*
	Simple prg to retrieve data from the Internet using GET /
HTTP/1.1.
	The whole prg is include in main().
*/
#include <stdio.h>
#include <netdb.h>
#include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h> // htons(), htonl()
#include <string.h>

#include <iostream.h>

int main(int argc, char *argv[])
{
	// Check for right # of arguments.
	if(argc > 2)
	{
		cout<< argv[0] <<" "<<"<URL>"<<endl;
		exit(1);
	}

	// Socket connection
	struct sockaddr_in sin;
	sin.sin_family	= AF_INET;
	sin.sin_port	= htons(80);
	
	struct hostent *host;
	host = gethostbyname(argv[1]);
	
	
	if(host == NULL)	// Unresolve host name.
	{
		fprintf(stderr, "Unresolve host name: gethostbename() failed");
		exit(1);
	}
	else
	{
		// Copy host address to sockaddr_in.sin_addr.s_addr.
		memcpy((caddr_t)& sin.sin_addr, host->h_addr, host->h_length);
	}

	
	// IPPROTO_TCP == 0.
	int sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
	
	if(sock < 0)	// -1, if socket() failed.
	{
		perror("socket() failed!");
		exit(1);
	}
	
	if(connect(sock, (struct sockaddr *) &sin, sizeof(sin)) < 0)
// -1 == error occured.
	{
		perror("connect() failed!");
		close(sock);
		exit(1);		
	}// 0 == no error occured.
	
	char szbuffer[1445];
	
	// Prepare the request string.
	strcpy(szbuffer, "GET / HTTP/1.0\nFrom:
very709394bad@yahoo.com\nUser-Agent: Testing/1.0\n\n ");
	//strcpy(szbuffer, "GET / HTTP/1.1\nHost:
www.yahoo.com\nConnection: Keep-Alive\nFrom:
very709394bad@yahoo.com\nUser-Agent: Mozilla/5.0\n\n ");
	//strcpy(szbuffer, "GET / HTTP/1.0\n\n ");

	// Send request.
	if(send(sock, szbuffer, strlen(szbuffer), MSG_OOB) < 0)
	{
		perror("send() failed!");
		exit(1);
	}

	// Clear szbuffer.
	memset(szbuffer, '\0', sizeof(szbuffer));

	// Open file to be written on.
	FILE *stream;
	if((stream = fopen("t.htm", "wb")) == NULL)
		perror("Can not write file!");
	
	int numBytesRecv=0, i=0;
	bool t=true;// For testing purpose ONLY
	while( (i = recv(sock, szbuffer, sizeof(szbuffer), 0)) > 0 )
	{
		numBytesRecv += fwrite(szbuffer, sizeof(char), i, stream);
		
		// For testing purpose ONLY
		if(t)
		{
			cout<<szbuffer<<endl<<"-----------"<<endl;
			t=false;
		}
		memset(szbuffer, '\0', sizeof(szbuffer));
		//printf("%i\n",i);
	}
	printf("Number of Bytes received: %i\n", numBytesRecv);

	return 0;
}
 
Reply


All articles

Subject Author Date
OFF Topic: HTTP protocol problem.

12/20/2002 21:25
Re: OFF Topic: HTTP protocol problem.

12/20/2002 21:35
Re: OFF Topic: HTTP protocol problem.

12/21/2002 01:11
Re: OFF Topic: HTTP protocol problem.

12/21/2002 06:19
Re: OFF Topic: HTTP protocol problem.

01/13/2006 14:31
Re: OFF Topic: HTTP protocol problem.

06/27/2009 03:27




0

Created with FORUM 2.0.11