大数据处理---C++

一个大文件，肯定是内存无法全部读取的，比如1TB，里边很多字段，假设是这样的：

1

1

1

这样一行一个数字，简单点，不统计每个数字的频率了，仅仅统计有多少个1，如何做呢？

FILE * p = fopen("test.dat","r");
__int64 len=0;
char a[2]={0};
for(int i = 0 ;; i ++)
{
	fread(a,2,1,p);
	if(a[0]!='1')
		break;
	len =i*1024;
}
printf("%s  %dn",a,len);
fclose(p);
p = NULL;

#include <stdio.h>
#include <Windows.h>
void createBigFile(int NumberGB=0)
{
	FILE * p = fopen("d:test.dat","w+");
	for(int i = 0 ; i < NumberGB ; i ++)
	{
		fwrite("1n",p);
	}
	fclose(p);
	p = NULL;
}
void CountBigFile0()
{
	FILE * p = fopen("test.dat","r");
	__int64 len=0;
	char a[2]={0};
	for(int i = 0 ; i < 2992742400 ; i ++)
	{
		fseek(p,i*1024,SEEK_SET);
		fread(a,p);
		if(a[0]!='1')
			break;
		len =i*1024;
	}
	printf("%s  %dn",len);
	fclose(p);
	p = NULL;
}
__int64 CountBigFile1(void)
{
	__int64 count = 0;
	SYSTEM_INFO sys;
	GetSystemInfo(&sys);
	HANDLE hFile = CreateFile(TEXT("d:test.dat"),FILE_READ_DATA|FILE_WRITE_DATA,FILE_SHARE_READ|FILE_SHARE_WRITE,NULL,OPEN_EXISTING,FILE_ATTRIBUTE_NORMAL,NULL);
	if(hFile == INVALID_HANDLE_VALUE) return 0;
	HANDLE hFileMapping = CreateFileMapping(hFile,PAGE_READWRITE | SEC_COMMIT,NULL);
	DWORD dwFileHigh;
	__int64 qwFileSize = GetFileSize(hFile,&dwFileHigh);
	qwFileSize += ( ((__int64)dwFileHigh)<<32);
	CloseHandle(hFile);
	__int64 qwFileOffset = 0;
	
	while (qwFileSize > 0)
	{
		DWORD dwBytesInblock = sys.dwAllocationGranularity;
		
		if(qwFileSize < dwBytesInblock)
			dwBytesInblock = qwFileSize;
		
		PBYTE pbFile = (PBYTE) MapViewOfFile(hFileMapping,FILE_MAP_WRITE|FILE_MAP_READ,(DWORD)qwFileOffset>>32,(DWORD)(qwFileOffset&0xffffffff),dwBytesInblock);

		memcpy(pbFile,"12345n",6);
		//for(int i = 0 ; i < dwBytesInblock ; i ++)
		//{
		//	//doing
		//	count ++;//记录行数
		//}

		//BOOL bRet = ::FlushViewOfFile(pbFile,dwBytesInblock);
		//if ( bRet == FALSE )
		//{
		//	DWORD dwError = GetLastError();
		//	return FALSE;
		//}
		//UnmapViewOfFile(pbFile);
		
		qwFileOffset+=dwBytesInblock;
		qwFileSize-=dwBytesInblock;
	}
	printf("count = %I64dn",qwFileSize);
	CloseHandle(hFileMapping);return count;
}
int main()
{
	createBigFile(20);
	CountBigFile1();
	return 0;
}

结果是一样的，很棒的大数据方法。

摘自图书：Windows via CC++ from vm?