diff --git a/index.php b/index.php index f400e6e..e66a33e 100644 --- a/index.php +++ b/index.php @@ -1,11 +1,11 @@ $name) { - $names[$key] = urlencode($name); - } - header('Location: sparql.php?names='. implode("|",$names)); - exit; - } - } + $result = $names; + if(isset($_POST['analyse']) && $_POST['analyse'] == 'false'){ + $names = normalizeSubmittedNames($names); + if (count($names)){ + foreach ($names as $key => $name) { + $names[$key] = urlencode($name); + } + header('Location: sparql.php?names='. implode("|",$names)); + exit; + } + } } elseif (isset($_POST['url']) && $_POST['url'] != '' ){ @@ -98,111 +98,111 @@ function handleFlickr($set_id){ $error .= $flickr['error']; } - elseif (is_404($url)) { - $error .= '404 page not found
'; - } - else{ - $result = extractTextFromUrl($url); - } -} -if ($result != ''){ - $names = extractCandidateNames($result, $nameFilters); - if(count($names) == 0){ + elseif (is_404($url)) { + $error .= '404 page not found
'; + } + else{ + $result = extractTextFromUrl($url); + } +} +if ($result != ''){ + $names = extractCandidateNames($result, $nameFilters); + if(count($names) == 0){ $error .= 'no content found
'; } } -elseif(isset($_POST['names']) && $_POST['names'] == ''|| isset($_POST['url']) && $_POST['url'] ==''){ - $error .= 'no input?'; -} - -function extractTextFromUrl($url) { - $readability = new Readability(new Configuration()); - $opts = array('http'=>array('header' => "User-Agent:Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0")); - $context = stream_context_create($opts); - $html = file_get_contents($url, false, $context); - $result = ''; - - try { - $readability->parse($html); - $result = $readability->getContent(); - } catch (ParseException $e) { - $result = extractTextWithPostlight($url); - } - - return $result == '' ? $html : $result; -} - -function extractTextWithPostlight($url) { - $ch = curl_init(); - $request_headers = array(); - $request_headers[] = 'x-api-key: ' . POSTLIGHTAPI; - $request_headers[] = "Content-Type: application/json; charset=utf-8"; - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); - curl_setopt($ch, CURLOPT_URL, 'https://mercury.postlight.com/parser?url=' . $url); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_HTTPHEADER, $request_headers); - $output = json_decode(curl_exec($ch)); - curl_close($ch); - - return isset($output->content) ? $output->content : ''; -} - -function extractCandidateNames($text, $filters) { - $text = prepareTextForNameExtraction($text, $filters); - - preg_match_all("/[\p{Lu}][\p{L}'\x{2019}\-]*[\p{L}](( ([\p{Lu}][\p{L}'\x{2019}\-]*[\p{L}]))*)( (([\p{Lu}]|Ph|Ch|Th)\.?)+)?(( ([\p{Lu}][\p{L}'\x{2019}\-]*[\p{L}]))*) ((van|de[rsn]?|van de[srn]?|el|(in)? ['\x{2019}]t|tot|te[rn]?|op|tot|uij?t|bij|aan|voor|von|Mac|\x{00D3}) )?([\p{Lu}][\p{L}'\x{2019}\-]*[\p{L}])+/u", $text, $matches); - if (!$matches) { - return array(); - } - - $names = array(); - foreach ($matches[0] as $name) { - $name = normalizeCandidateName($name); - if ($name != '' && !rejectCandidateName($name, $filters)) { - $names[] = $name; - } - } - $names = array_iunique($names); - $names = array_filter($names); - sort($names); - - return $names; -} - -function prepareTextForNameExtraction($text, $filters) { - $text = str_replace($filters['htmlBreakTags'], " \n", $text); - foreach ($filters['prefixWords'] as $value) { - $text = preg_replace("/\b".$value." \b/m", " ", $text); - } - - return html_entity_decode($text); -} - -function normalizeCandidateName($name) { - $name = trim($name); - $name = preg_replace("/^((Prof|Dr|Mr|Ms)\.?)?( ?(Prof|Dr|Mr|Ms)\.?)? (.+)/", "$5", $name); - return trim($name); -} - -function rejectCandidateName($name, $filters) { - foreach ($filters['rejectWords'] as $item) { - if(preg_match("/\b".$item."\b/", strtoupper($name))){ - return true; - } - } - - return false; -} - -function normalizeSubmittedNames($names) { - $names = explode("\n", $names); - $names = array_map('trim', $names); - $names = array_filter($names); - return array_iunique($names); -} - -function is_404($url) { +elseif(isset($_POST['names']) && $_POST['names'] == ''|| isset($_POST['url']) && $_POST['url'] ==''){ + $error .= 'no input?'; +} + +function extractTextFromUrl($url) { + $readability = new Readability(new Configuration()); + $opts = array('http'=>array('header' => "User-Agent:Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0")); + $context = stream_context_create($opts); + $html = file_get_contents($url, false, $context); + $result = ''; + + try { + $readability->parse($html); + $result = $readability->getContent(); + } catch (ParseException $e) { + $result = extractTextWithPostlight($url); + } + + return $result == '' ? $html : $result; +} + +function extractTextWithPostlight($url) { + $ch = curl_init(); + $request_headers = array(); + $request_headers[] = 'x-api-key: ' . POSTLIGHTAPI; + $request_headers[] = "Content-Type: application/json; charset=utf-8"; + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($ch, CURLOPT_URL, 'https://mercury.postlight.com/parser?url=' . $url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_HTTPHEADER, $request_headers); + $output = json_decode(curl_exec($ch)); + curl_close($ch); + + return isset($output->content) ? $output->content : ''; +} + +function extractCandidateNames($text, $filters) { + $text = prepareTextForNameExtraction($text, $filters); + + preg_match_all("/[\p{Lu}][\p{L}'\x{2019}\-]*[\p{L}](( ([\p{Lu}][\p{L}'\x{2019}\-]*[\p{L}]))*)( (([\p{Lu}]|Ph|Ch|Th)\.?)+)?(( ([\p{Lu}][\p{L}'\x{2019}\-]*[\p{L}]))*) ((van|de[rsn]?|van de[srn]?|el|(in)? ['\x{2019}]t|tot|te[rn]?|op|tot|uij?t|bij|aan|voor|von|Mac|\x{00D3}) )?([\p{Lu}][\p{L}'\x{2019}\-]*[\p{L}])+/u", $text, $matches); + if (!$matches) { + return array(); + } + + $names = array(); + foreach ($matches[0] as $name) { + $name = normalizeCandidateName($name); + if ($name != '' && !rejectCandidateName($name, $filters)) { + $names[] = $name; + } + } + $names = array_iunique($names); + $names = array_filter($names); + sort($names); + + return $names; +} + +function prepareTextForNameExtraction($text, $filters) { + $text = str_replace($filters['htmlBreakTags'], " \n", $text); + foreach ($filters['prefixWords'] as $value) { + $text = preg_replace("/\b".$value." \b/m", " ", $text); + } + + return html_entity_decode($text); +} + +function normalizeCandidateName($name) { + $name = trim($name); + $name = preg_replace("/^((Prof|Dr|Mr|Ms)\.?)?( ?(Prof|Dr|Mr|Ms)\.?)? (.+)/", "$5", $name); + return trim($name); +} + +function rejectCandidateName($name, $filters) { + foreach ($filters['rejectWords'] as $item) { + if(preg_match("/\b".$item."\b/", strtoupper($name))){ + return true; + } + } + + return false; +} + +function normalizeSubmittedNames($names) { + $names = explode("\n", $names); + $names = array_map('trim', $names); + $names = array_filter($names); + return array_iunique($names); +} + +function is_404($url) { $handle = curl_init($url); curl_setopt($handle, CURLOPT_FOLLOWLOCATION, TRUE); curl_setopt($handle, CURLOPT_MAXREDIRS, 5); @@ -279,7 +279,7 @@ function array_iunique($array) {
- Search for names in on the following webpage: + Search for names on the following webpage:
@@ -315,7 +315,7 @@ function array_iunique($array) { } ?> - +
@@ -326,4 +326,4 @@ function array_iunique($array) { - +