HTTrack Website Copier
Free software offline browser - FORUM
Subject: Can't get consistent working
Author: Brett G
Date: 12/28/2006 23:50
 
Hello.  I'm having a lot of trouble getting HTTrack to work in a consistent
way.  Depending on where I add the callbacks and call init, it will either
work correctly the first time and not work right after that, or it will not
work the first time and will work after that.  What I mean by "work correctly"
is that all of the expected callbacks will be called and the page and files
downloaded.  Incorrectly means that the callbacks are not called and it just
"hangs".

I'm using a background thread for downloading.  I've tried it using both
CreateThread and hts_newthread.

I have included my very simple code below.  You can see where I conditionally
compile the code for adding the callbacks.

I am further frustrated because cannot seem to find the documentation for the
APIs.  I don't know what methods I should call and what they mean.  I
especially want to know about the "loop" callback and its parameters.

Thanks for any help.  Here is the code.

----------------------
#include "stdafx.h"
#include "httrack.h"
#include "DownloadProgressDlg.h"
extern "C" 
{ 
#include "httrack-library.h" 
#include "htsthread.h" 
}


// HTTPRACK callbacks
int   __cdecl httrackengine_check(char* adr,char* fil,int status);
int   __cdecl httrackengine_check_mime(char* adr,char* fil,char* mime,int
status);
void  __cdecl httrackengine_init();
void  __cdecl httrackengine_uninit();
int   __cdecl httrackengine_start(void* dummy);
int   __cdecl httrackengine_end();
int   __cdecl httrackengine_htmlpreprocess(char** html,int* len,char*
url_adresse,char* url_fichier);
int   __cdecl httrackengine_htmlpostprocess(char** html,int* len,char*
url_adresse,char* url_fichier);
int   __cdecl httrackengine_htmlcheck(char* html,int len,char*
url_adresse,char* url_fichier);
int   __cdecl httrackengine_chopt(void* opt);
int   __cdecl httrackengine_loop(void* _back,int back_max,int back_index,int
lien_n,int lien_tot,int stat_time,hts_stat_struct* stats);
char* __cdecl httrackengine_query(char* question);
char* __cdecl httrackengine_query2(char* question);
char* __cdecl httrackengine_query3(char* question);
void  __cdecl httrackengine_pause(char* lockfile);
void  __cdecl httrackengine_filesave(char* file);
void  __cdecl httrackengine_filesave2(char* adr, char* fil, char* save, int
is_new, int is_modified, int not_updated);

DownloadInfo	*f_pDlInfo;
EZTOOLS::CCodeLock	f_codeLock;
HWND			f_hwndProgress;

#define HTS_THREAD

#ifdef HTS_THREAD
void __cdecl DownloadThreadProc( LPVOID lpParameter )
#else
DWORD WINAPI DownloadThreadProc( LPVOID lpParameter )
#endif
{
//	termine=termine_requested=shell_terminated=soft_term_requested=0;

	TBuf<char*> *pParams = (TBuf<char*>*) lpParameter;
	int nResult, argc;

	for( argc=0; argc < pParams->size(); argc++ )
		if( NOT (*pParams)[argc] )
			break;

    try
	{
		hts_init();
		//hts_resetvar();

		#if 1
		//htswrap_init();
		htswrap_add("check-link",httrackengine_check);
		htswrap_add("check-mime",httrackengine_check_mime);
		//htswrap_add("init",httrackengine_init);
		htswrap_add("free",httrackengine_uninit);
		htswrap_add("start",httrackengine_start);
		htswrap_add("end",httrackengine_end);
		htswrap_add("preprocess-html",httrackengine_htmlpreprocess);
		htswrap_add("postprocess-html",httrackengine_htmlpostprocess);
		htswrap_add("check-html",httrackengine_htmlcheck);
		htswrap_add("change-options",httrackengine_chopt);
		htswrap_add("loop",httrackengine_loop);
		htswrap_add("query",httrackengine_query);
		htswrap_add("query2",httrackengine_query2);
		htswrap_add("query3",httrackengine_query3);
		htswrap_add("pause",httrackengine_pause);
		htswrap_add("save-file",httrackengine_filesave);
		htswrap_add("save-file2",httrackengine_filesave2);
#endif

		nResult = hts_main( argc, *pParams );
	}
	catch(...) // ExcFilter_(GetExceptionCode(), GetExceptionInformation())
	{
		nResult = -100;
	}

	#ifdef HTS_THREAD
    htsthread_wait_n(1);
	#endif
    hts_uninit();

	PostMessage( f_hwndProgress, WM_COMMAND, MAKEWPARAM(IDOK,BN_CLICKED), NULL
);

#ifndef HTS_THREAD
	return 0;
#endif
}

void DoDownload( DownloadInfo &dlInfo )
{
	static char* ppArgv[] =
	{
		"DownZip",
		//"-qwC2%Ps2u1%s%uN1%I0p7DaK0H0%kf2A25000%f#f", // create err file for
missing files
		"-qwC2%Ps2u1%s%uN1%I0p7DaK0H0%kf2o0A25000%f#f",	// no err files
		"-F",
		"Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)",
		"-%F",
		"<!-- Mirrored from %s%s by DownZip Website Copier/1.x [XR&CO'2006], %s
-->",
		"-%l",
		"en, en, *"
		, <http://www.wincentric.com>,
		"-O1",
		"C:\\Documents and Settings\\Brett\\My Documents\\My Webpages\\WinCentric",
		"+*.png", "+*.gif", "+*.jpg", "+*.css", "+*.js",
		"-*.htm*", "-*.asp*" "-*.php*", "-*.exe", "-*.swf", "-*.zip",
		"-ad.doubleclick.net/*", "-mime:application/foobar"
	};

	f_pDlInfo = &dlInfo;

	hts_init();
	hts_resetvar();

#if 0
	htswrap_init();
	htswrap_add("check-link",httrackengine_check);
	htswrap_add("check-mime",httrackengine_check_mime);
	//htswrap_add("init",httrackengine_init);
	htswrap_add("free",httrackengine_uninit);
	htswrap_add("start",httrackengine_start);
	htswrap_add("end",httrackengine_end);
	htswrap_add("preprocess-html",httrackengine_htmlpreprocess);
	htswrap_add("postprocess-html",httrackengine_htmlpostprocess);
	htswrap_add("check-html",httrackengine_htmlcheck);
	htswrap_add("change-options",httrackengine_chopt);
	htswrap_add("loop",httrackengine_loop);
	htswrap_add("query",httrackengine_query);
	htswrap_add("query2",httrackengine_query2);
	htswrap_add("query3",httrackengine_query3);
	htswrap_add("pause",httrackengine_pause);
	htswrap_add("save-file",httrackengine_filesave);
	htswrap_add("save-file2",httrackengine_filesave2);
#endif

	int n, nArgs;
	TBuf<char*> params(0);

	params.reallocate( 100, false, true ); // to zero the memory

	for( n=0, nArgs=NUM_ELEMENTS(ppArgv); n < nArgs; n++ )
	{
		if( n >= params.size() ) // increase size
			params.reallocate( params.size()*3/2, true );
		params[n] = ppArgv[n];
	}

	DWORD dwThreadId=0;
	HANDLE hThread=0;
	#ifdef HTS_THREAD
	hts_newthread( DownloadThreadProc, 0, &params );
	#else
	hThread = CreateThread( NULL, 0, DownloadThreadProc, &params,
CREATE_SUSPENDED, &dwThreadId );
	#endif

	CDownloadProgressDlg progressDlg( hThread, dlInfo );
	f_hwndProgress = progressDlg.m_hWnd;

	progressDlg.DoModal();

}

int __cdecl httrackengine_check(char* adr,char* fil,int status)
{
	return -1;
}

int __cdecl httrackengine_check_mime(char* adr,char* fil,char* mime,int
status)
{
	return -1;
}

EXECUTION_STATE (WINAPI * SetThreadExecutionState_)(IN EXECUTION_STATE) =
NULL;

void __cdecl httrackengine_init()
{
}

int __cdecl httrackengine_start(void* dummy)
{
	httrackengine_loop( NULL,0,0,0,0,NULL,0 );  // init
	return 1;
}

void __cdecl httrackengine_uninit()
{
}

int httrackengine_end()
{
	EZTOOLS::CCodeLockSentry lockSentry(f_codeLock);

	f_pDlInfo->m_bTerminate = TRUE;

	return 1;
}

int __cdecl httrackengine_htmlpreprocess(char** html,int* len,char*
url_adresse,char* url_fichier)
{
	return 1;
}

int __cdecl httrackengine_htmlpostprocess(char** html,int* len,char*
url_adresse,char* url_fichier)
{
	return 1;
}

int __cdecl httrackengine_htmlcheck(char* html,int len,char* url_adresse,char*
url_fichier)
{
	return 1;
}

int __cdecl httrackengine_chopt(void* opt)
{
	ATLTRACE(__FUNCTION__ " : changing options\r\n");
	return 1;
}

void __cdecl httrackengine_filesave(char* file)
{
}

void __cdecl httrackengine_filesave2(char* adr, char* fil, char* save, int
is_new, int is_modified, int not_updated)
{
}

// Le routine la plus utile sans doute: elle refresh les tableaux
// C'est la 2e routine en thread qui assure le refresh graphique
// (plus efficace)
// -->C'est elle qui décide de tout arrêter si elle détecte in
termine_request<--
//
int __cdecl httrackengine_loop(
							   void* _back,int back_max,int back_index,
							   int lien_n,int lien_tot,
							   int stat_time,
							   hts_stat_struct* stats)
{
	EZTOOLS::CCodeLockSentry lockSentry(f_codeLock);
	if( f_pDlInfo->m_bCancel OR f_pDlInfo->m_bTerminate )
		return 0;
	lockSentry.Unlock();

	// TODO: update UI

	return 1; // continue
}

char* __cdecl httrackengine_query(char* question)
{
	return "";
}

char* __cdecl httrackengine_query2(char* question)
{
	return "";
}

char* __cdecl httrackengine_query3(char* question)
{
	return "";
}

void __cdecl httrackengine_pause(char* lockfile)
{
}
 
Reply


All articles

Subject Author Date
Can't get consistent working

12/28/2006 23:50
Re: Can't get consistent working

12/29/2006 01:02




6

Created with FORUM 2.0.11