?? RIGHT := 110 ??
?? NEWTITLE := 'NOS/VE SCL Interpreter : Lexical Processors' ??
MODULE clm$f_lexical_processors;

{
{ PURPOSE:
{   This module contains an interface between the SCL formatter routines
{   and clp$scan_lexical_unit. It provides for generating a "format token"
{   for each clt$lexical_unit encountered.
{

?? NEWTITLE := 'Global Declarations', EJECT ??
?? PUSH (LISTEXT := ON) ??
*copyc clt$parse_state
*copyc clt$slu_termination_option
*copyc oss$job_paged_literal
?? POP ??
*copyc clp$add_format_token
*copyc clp$delete_current_format_token
*copyc clp$scan_lexical_unit

  VAR
    spaces_before_not_part_of_token: [STATIC, READ, oss$job_paged_literal] set of clt$lexical_unit_kind :=
          [clc$lex_unknown, clc$lex_dot, clc$lex_colon, clc$lex_left_parenthesis, clc$lex_query, clc$lex_add,
          clc$lex_subtract, clc$lex_string, clc$lex_name, clc$lex_unsigned_decimal, clc$lex_alpha_number];

?? TITLE := 'clp$f_scan_token', EJECT ??

  PROCEDURE [XDCL] clp$f_scan_token
    (    termination_option: clt$slu_termination_option;
     VAR parse { input, output } : clt$parse_state);

    VAR
      last_token_space: boolean;

    CASE termination_option OF
    = clc$slu_any =
      clp$scan_lexical_unit (termination_option, parse);
      IF parse.unit.size > 0 THEN
        clp$add_format_token (^parse.text^ (parse.unit_index, parse.unit.size), parse.unit.kind,
              clc$unassigned);
      IFEND;
    = clc$slu_non_space =
      last_token_space := FALSE;

    /find_non_space/
      WHILE TRUE DO
        clp$scan_lexical_unit (clc$slu_any, parse);
        IF NOT (parse.unit_is_space) THEN
          IF last_token_space AND NOT (parse.unit.kind IN spaces_before_not_part_of_token) THEN
            clp$delete_current_format_token;
          IFEND;
          EXIT /find_non_space/;
        IFEND;
        last_token_space := parse.unit.kind = clc$lex_space;
        clp$add_format_token (^parse.text^ (parse.unit_index, parse.unit.size), parse.unit.kind,
              clc$unassigned);
      WHILEND /find_non_space/;

      IF parse.unit.size > 0 THEN
        clp$add_format_token (^parse.text^ (parse.unit_index, parse.unit.size), parse.unit.kind,
              clc$unassigned);
      IFEND;

    CASEND;

  PROCEND clp$f_scan_token;

MODEND clm$f_lexical_processors;
